fix mounts for pre-existing subcontainers (#2870)

* fix mounts for pre-existing subcontainers

* don't error on missing assets

.github/workflows/startos-iso.yaml

@@ -12,9 +12,6 @@ on:
           - dev
           - unstable
           - dev-unstable
-          - docker
-          - dev-docker
-          - dev-unstable-docker
       runner:
         type: choice
         description: Runner
@@ -41,14 +38,14 @@ on:
   push:
     branches:
       - master
-      - next
+      - next/*
   pull_request:
     branches:
       - master
-      - next
+      - next/*
 
 env:
-  NODEJS_VERSION: "18.15.0"
+  NODEJS_VERSION: "20.16.0"
   ENVIRONMENT: '${{ fromJson(format(''["{0}", ""]'', github.event.inputs.environment || ''dev''))[github.event.inputs.environment == ''NONE''] }}'
 
 jobs:
@@ -74,24 +71,32 @@ jobs:
           sudo mount -t tmpfs tmpfs .
         if: ${{ github.event.inputs.runner == 'fast' }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
 
-      - uses: actions/setup-node@v3
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODEJS_VERSION }}
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      - name: Set up docker QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up system dependencies
+        run: sudo apt-get update && sudo apt-get install -y qemu-user-static systemd-container squashfuse
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Make
         run: make ARCH=${{ matrix.arch }} compiled-${{ matrix.arch }}.tar
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: compiled-${{ matrix.arch }}.tar
           path: compiled-${{ matrix.arch }}.tar
@@ -140,10 +145,19 @@ jobs:
           }')[matrix.platform]
         }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Free space
+        run: rm -rf /opt/hostedtoolcache*
+        if: ${{ github.event.inputs.runner != 'fast' }}
+
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
 
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
       - name: Install dependencies
         run: |
           sudo apt-get update
@@ -162,7 +176,7 @@ jobs:
         if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree') }}
 
       - name: Download compiled artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: compiled-${{ env.ARCH }}.tar
 
@@ -171,9 +185,29 @@ jobs:
 
       - name: Prevent rebuild of compiled artifacts
         run: |
-          mkdir -p frontend/dist/raw
+          mkdir -p web/node_modules
+          mkdir -p web/dist/raw
+          mkdir -p core/startos/bindings
+          mkdir -p sdk/base/lib/osBindings
+          mkdir -p container-runtime/node_modules
+          mkdir -p container-runtime/dist
+          mkdir -p container-runtime/dist/node_modules
+          mkdir -p core/startos/bindings
+          mkdir -p sdk/dist
+          mkdir -p sdk/baseDist
+          mkdir -p patch-db/client/node_modules
+          mkdir -p patch-db/client/dist
+          mkdir -p web/.angular
+          mkdir -p web/dist/raw/ui
+          mkdir -p web/dist/raw/install-wizard
+          mkdir -p web/dist/raw/setup-wizard
+          mkdir -p web/dist/static/ui
+          mkdir -p web/dist/static/install-wizard
+          mkdir -p web/dist/static/setup-wizard
           PLATFORM=${{ matrix.platform }} make -t compiled-${{ env.ARCH }}.tar
 
+      - run: git status
+
       - name: Run iso build
         run: PLATFORM=${{ matrix.platform }} make iso
         if: ${{ matrix.platform != 'raspberrypi' }}
@@ -182,18 +216,18 @@ jobs:
         run: PLATFORM=${{ matrix.platform }} make img
         if: ${{ matrix.platform == 'raspberrypi' }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.platform }}.squashfs
           path: results/*.squashfs
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.platform }}.iso
           path: results/*.iso
         if: ${{ matrix.platform != 'raspberrypi' }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.platform }}.img
           path: results/*.img

.github/workflows/test.yaml

@@ -0,0 +1,31 @@
+name: Automated Tests
+
+on:
+  push:
+    branches:
+      - master
+      - next/*
+  pull_request:
+    branches:
+      - master
+      - next/*
+
+env:
+  NODEJS_VERSION: "20.16.0"
+  ENVIRONMENT: dev-unstable
+
+jobs:
+  test:
+    name: Run Automated Tests
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODEJS_VERSION }}
+
+      - name: Build And Run Tests
+        run: make test

.gitignore

@@ -20,11 +20,12 @@ secrets.db
 /ENVIRONMENT.txt
 /GIT_HASH.txt
 /VERSION.txt
-/eos-*.tar.gz
 /*.deb
 /target
 /*.squashfs
 /results
 /dpkg-workdir
 /compiled.tar
-/compiled-*.tar
+/compiled-*.tar
+/firmware
+/tmp

CHANGELOG.md

@@ -1,429 +0,0 @@
-# v0.3.3
-## Highlights
-    - x86_64 architecture compatibility
-    - Kiosk mode - use your Embassy with monitor, keyboard, and mouse (available on x86 builds only, disabled on Raspberry Pi)
-    - "Updates" tab - view all service updates from all registries in one place
-    - Various UI/UX improvements
-    - Various bugfixes and optimizations
-
-## What's Changed
-    - Minor typo fixes by @kn0wmad in #1887
-    - Update build pipeline by @moerketh in #1896
-    - Feature/setup migrate by @elvece in #1841
-    - Feat/patch migration by @Blu-J in #1890
-    - make js cancellable by @dr-bonez in #1901
-    - wip: Making Injectable exec by @Blu-J in #1897
-    - Fix/debug by @Blu-J in #1909
-    - chore: Fix on the rsync not having stdout. by @Blu-J in #1911
-    - install wizard project by @MattDHill in #1893
-    - chore: Remove the duplicate loggging information that is making usele… by @Blu-J in #1912
-    - Http proxy by @redragonx in #1772
-    - fix(marketplace): loosen type in categories component by @waterplea in #1918
-    - set custom meta title by @MattDHill in #1915
-    - Feature/git hash by @dr-bonez in #1919
-    - closes #1900 by @dr-bonez in #1920
-    - feature/marketplace icons by @dr-bonez in #1921
-    - Bugfix/0.3.3 migration by @dr-bonez in #1922
-    - feat: Exposing the rsync that we have to the js by @Blu-J in #1907
-    - Feature/install wizard disk info by @dr-bonez in #1923
-    - bump shared and marketplace npm versions by @dr-bonez in #1924
-    - fix error handling when store unreachable by @dr-bonez in #1925
-    - wait for network online before launching init by @dr-bonez in #1930
-    - silence service crash notifications by @dr-bonez in #1929
-    - disable efi by @dr-bonez in #1931
-    - Tor daemon fix by @redragonx in #1934
-    - wait for url to be available before launching kiosk by @dr-bonez in #1933
-    - fix migration to support portable fatties by @dr-bonez in #1935
-    - Add guid to partition type by @MattDHill in #1932
-    - add localhost support to the http server by @redragonx in #1939
-    - refactor setup wizard by @dr-bonez in #1937
-    - feat(shared): Ticker add new component and use it in marketplace by @waterplea in #1940
-    - feat: For ota update using rsyncd by @Blu-J in #1938
-    - Feat/update progress by @MattDHill in #1944
-    - Fix/app show hidden by @MattDHill in #1948
-    - create dpkg and iso workflows by @dr-bonez in #1941
-    - changing ip addr type by @redragonx in #1950
-    - Create mountpoints first by @k0gen in #1949
-    - Hard code registry icons by @MattDHill in #1951
-    - fix: Cleanup by sending a command and kill when dropped by @Blu-J in #1945
-    - Update setup wizard styling by @elvece in #1954
-    - Feature/homepage by @elvece in #1956
-    - Fix millis by @Blu-J in #1960
-    - fix accessing dev tools by @MattDHill in #1966
-    - Update/misc UI fixes by @elvece in #1961
-    - Embassy-init typo by @redragonx in #1959
-    - feature: 0.3.2 -> 0.3.3 upgrade by @dr-bonez in #1958
-    - Fix/migrate by @Blu-J in #1962
-    - chore: Make validation reject containers by @Blu-J in #1970
-    - get pubkey and encrypt password on login by @elvece in #1965
-    - Multiple bugs and styling by @MattDHill in #1975
-    - filter out usb stick during install by @dr-bonez in #1974
-    - fix http upgrades by @dr-bonez in #1980
-    - restore interfaces before creating manager by @dr-bonez in #1982
-    - fuckit: no patch db locks by @dr-bonez in #1969
-    - fix websocket hangup error by @dr-bonez in #1981
-    - revert app show to use header and fix back button by @MattDHill in #1984
-    - Update/marketplace info by @elvece in #1983
-    - force docker image removal by @dr-bonez in #1985
-    - do not error if cannot determine live usb device by @dr-bonez in #1986
-    - remove community registry from FE defaults by @MattDHill in #1988
-    - check environment by @dr-bonez in #1990
-    - fix marketplace search and better category disabling by @MattDHill in #1991
-    - better migration progress bar by @dr-bonez in #1993
-    - bump cargo version by @dr-bonez in #1995
-    - preload icons and pause on setup complete for kiosk mode by @MattDHill in #1997
-    - use squashfs for rpi updates by @dr-bonez in #1998
-    - do not start progress at 0 before diff complete by @dr-bonez in #1999
-    - user must click continue in kiosk on success page by @MattDHill in #2001
-    - fix regex in image rip script by @dr-bonez in #2002
-    - fix bug with showing embassy drives and center error text by @MattDHill in #2006
-    - fix partition type by @dr-bonez in #2007
-    - lowercase service for alphabetic sorting by @MattDHill in #2008
-    - dont add updates cat by @MattDHill in #2009
-    - make downloaded page a full html doc by @MattDHill in #2011
-    - wait for monitor to be attached before launching firefox by @chrisguida in #2005
-    - UI fixes by @elvece in #2014
-    - fix: Stop service before by @Blu-J in #2019
-    - shield links update by @k0gen in #2018
-    - fix: Undoing the breaking introduced by trying to stopp by @Blu-J in #2023
-    - update link rename from embassy -> system by @elvece in #2027
-    - initialize embassy before restoring packages by @dr-bonez in #2029
-    - make procfs an optional dependency so sdk can build on macos by @elvece in #2028
-    - take(1) for recover select by @MattDHill in #2030
-    - take one from server info to prevent multiple reqs to registries by @MattDHill in #2032
-    - remove write lock during backup by @MattDHill in #2033
-    - fix: Ensure that during migration we make the urls have a trailing slash by @Blu-J in #2036
-    - fix: Make the restores limited # restore at a time by @Blu-J in #2037
-    - fix error and display of unknown font weight on success page by @elvece in #2038
-
-## Checksums
-```
-8602e759d3ece7cf503b9ca43e8419109f14e424617c2703b3771c8801483d7e  embassyos_amd64.deb
-b5c0d8d1af760881a1b5cf32bd7c5b1d1cf6468f6da594a1b4895a866d03a58c  embassyos_amd64.iso
-fe518453a7e1a8d8c2be43223a1a12adff054468f8082df0560e1ec50df3dbfd  embassyos_raspberrypi.img
-7b1ff0ada27b6714062aa991ec31c2d95ac4edf254cd464a4fa251905aa47ebd  embassyos_raspberrypi.tar.gz
-```
-
-# v0.3.2.1
-## What's Changed
-    - Update index.html copy and styling by @elvece in #1855
-    - increase maximum avahi entry group size by @dr-bonez in #1869
-    - bump version by @dr-bonez in #1871
-
-### Linux and Mac
-
-Download the `eos.tar.gz` file, then extract and flash the resulting eos.img to your SD Card
-Windows
-
-Download the `eos.zip` file, then extract and flash the resulting eos.img to your SD Card
-
-## SHA-256 Checksums
-```
-c4b17658910dd10c37df134d5d5fdd6478f962ba1b803d24477d563d44430f96  eos.tar.gz
-3a8b29878fe222a9d7cbf645c975b12805704b0f39c7daa46033d22380f9828c eos.zip
-dedff3eb408ea411812b8f46e6c6ed32bfbd97f61ec2b85a6be40373c0528256  eos.img
-```
-
-# v0.3.2
-## Highlights
-    - Autoscrolling for logs
-    - Improved connectivity between browser and Embassy
-    - Switch to Postgres for EOS database for better performance
-    - Multiple bug fixes and under-the-hood improvements
-    - Various UI/UX enhancements
-    - Removal of product keys
-
-Update Hash (SHA256): `d8ce908b06baee6420b45be1119e5eb9341ba8df920d1e255f94d1ffb7cc4de9`
-
-Image Hash (SHA256): `e035cd764e5ad9eb1c60e2f7bc3b9bd7248f42a91c69015c8a978a0f94b90bbb`
-
-Note: This image was uploaded as a gzipped POSIX sparse TAR file. The recommended command for unpacking it on systems that support sparse files is `tar --format=posix --sparse -zxvf eos.tar.gz`
-
-## What's Changed
-    - formatting by @dr-bonez in #1698
-    - Update README.md by @kn0wmad in #1705
-    - Update README.md by @dr-bonez in #1703
-    - feat: migrate to Angular 14 and RxJS 7 by @waterplea in #1681
-    - 0312 multiple FE by @MattDHill in #1712
-    - Fix http requests by @MattDHill in #1717
-    - Add build-essential to README.md by @chrisguida in #1716
-    - write image to sparse-aware archive format by @dr-bonez in #1709
-    - fix: Add modification to the max_user_watches by @Blu-J in #1695
-    - [Feat] follow logs by @chrisguida in #1714
-    - Update README.md by @dr-bonez in #1728
-    - fix build for patch-db client for consistency by @elvece in #1722
-    - fix cli install by @chrisguida in #1720
-    - highlight instructions if not viewed by @MattDHill in #1731
-    - Feat: HttpReader by @redragonx in #1733
-    - Bugfix/dns by @dr-bonez in #1741
-    - add x86 build and run unittests to backend pipeline by @moerketh in #1682
-    - [Fix] websocket connecting and patchDB connection monitoring by @MattDHill in #1738
-    - Set pipeline job timeouts and add ca-certificates to test container by @moerketh in #1753
-    - Disable bluetooth properly #862 by @redragonx in #1745
-    - [feat]: resumable downloads by @dr-bonez in #1746
-    - Fix/empty properties by @elvece in #1764
-    - use hostname from patchDB as default server name by @MattDHill in #1758
-    - switch to postgresql by @dr-bonez in #1763
-    - remove product key from setup flow by @MattDHill in #1750
-    - pinning cargo dep versions for CLI by @redragonx in #1775
-    - fix: Js deep dir by @Blu-J in #1784
-    - 0.3.2 final cleanup by @dr-bonez in #1782
-    - expect ui marketplace to be undefined by @MattDHill in #1787
-    - fix init to exit on failure by @dr-bonez in #1788
-    - fix search to return more accurate results by @MattDHill in #1792
-    - update backend dependencies by @dr-bonez in #1796
-    - use base64 for HTTP headers by @dr-bonez in #1795
-    - fix: Bad cert of *.local.local is now fixed to correct. by @Blu-J in #1798
-    - fix duplicate patch updates, add scroll button to setup success by @MattDHill in #1800
-    - level_slider reclaiming that precious RAM memory by @k0gen in #1799
-    - stop leaking avahi clients by @dr-bonez in #1802
-    - fix: Deep is_parent was wrong and could be escapped by @Blu-J in #1801
-    - prevent cfg str generation from running forever by @dr-bonez in #1804
-    - better RPC error message by @MattDHill in #1803
-    - Bugfix/marketplace add by @elvece in #1805
-    - fix mrketplace swtiching by @MattDHill in #1810
-    - clean up code and logs by @MattDHill in #1809
-    - fix: Minor fix that matt wanted by @Blu-J in #1808
-    - onion replace instead of adding tor repository by @k0gen in #1813
-    - bank Start as embassy hostname from the begining by @k0gen in #1814
-    - add descriptions to marketplace list page by @elvece in #1812
-    - Fix/encryption by @elvece in #1811
-    - restructure initialization by @dr-bonez in #1816
-    - update license by @MattDHill in #1819
-    - perform system rebuild after updating by @dr-bonez in #1820
-    - ignore file not found error for delete by @dr-bonez in #1822
-    - Multiple by @MattDHill in #1823
-    - Bugfix/correctly package backend job by @moerketh in #1826
-    - update patch-db by @dr-bonez in #1831
-    - give name to logs file by @MattDHill in #1833
-    - play song during update by @dr-bonez in #1832
-    - Seed patchdb UI data by @elvece in #1835
-    - update patch db and enable logging by @dr-bonez in #1837
-    - reduce patch-db log level to warn by @dr-bonez in #1840
-    - update ts matches to fix properties ordering bug by @elvece in #1843
-    - handle multiple image tags having the same hash and increase timeout by @dr-bonez in #1844
-    - retry pgloader up to 5x by @dr-bonez in #1845
-    - show connection bar right away by @MattDHill in #1849
-    - dizzy Rebranding to embassyOS by @k0gen in #1851
-    - update patch db by @MattDHill in #1852
-    - camera_flash screenshots update by @k0gen in #1853
-    - disable concurrency and delete tmpdir before retry by @dr-bonez in #1846
-
-## New Contributors
-
-	- @redragonx made their first contribution in #1733
-
-
-# v0.3.1.1
-## What's Changed
-
-    - whale2 docker stats fix by @k0gen in #1630
-    - update backend dependencies by @dr-bonez in #1637
-    - Fix/receipts health by @Blu-J in #1616
-    - return correct error on failed os download by @dr-bonez in #1636
-    - fix build by @dr-bonez in #1639
-    - Update product.yaml by @dr-bonez in #1638
-    - handle case where selected union enum is invalid after migration by @MattDHill in #1658
-    - fix: Resolve fighting with NM by @Blu-J in #1660
-    - sdk: don't allow mounts in inject actions by @chrisguida in #1653
-    - feat: Variable args by @Blu-J in #1667
-    - add readme to system-images folder by @elvece in #1665
-    - Mask chars beyond 16 by @MattDHill in #1666
-    - chore: Update to have the new version 0.3.1.1 by @Blu-J in #1668
-    - feat: Make the rename effect by @Blu-J in #1669
-    - fix migration, add logging by @dr-bonez in #1674
-    - run build checks only when relevant FE changes by @elvece in #1664
-    - trust local ca by @dr-bonez in #1670
-    - lower log level for docker deser fallback message by @dr-bonez in #1672
-    - refactor build process by @dr-bonez in #1675
-    - chore: enable strict mode by @waterplea in #1569
-    - draft releases notes for 0311 by @MattDHill in #1677
-    - add standby mode by @dr-bonez in #1671
-    - feat: atomic writing by @Blu-J in #1673
-    - allow server.update to update to current version by @dr-bonez in #1679
-    - allow falsey rpc response by @dr-bonez in #1680
-    - issue notification when individual package restore fails by @dr-bonez in #1685
-    - replace bang with question mark in html by @MattDHill in #1683
-    - only validate mounts for inject if eos >=0.3.1.1 by @dr-bonez in #1686
-    - add marketplace_url to backup metadata for service by @dr-bonez in #1688
-    - marketplace published at for service by @MattDHill in #1689
-    - sync data to fs before shutdown by @dr-bonez in #1690
-    - messaging for restart, shutdown, rebuild by @MattDHill in #1691
-    - honor shutdown from diagnostic ui by @dr-bonez in #1692
-    - ask for sudo password immediately during make by @dr-bonez in #1693
-    - sync blockdev after update by @dr-bonez in #1694
-    - set Matt as default assignee by @MattDHill in #1697
-    - NO_KEY for CI images by @dr-bonez in #1700
-    - fix typo by @dr-bonez in #1702
-
-# v0.3.1
-## What's Changed
-    - Feat bulk locking by @Blu-J in #1422
-    - Switching SSH keys to start9 user by @k0gen in #1321
-    - chore: Convert from ajv to ts-matches by @Blu-J in #1415
-    - Fix/id params by @elvece in #1414
-    - make nicer update sound by @ProofOfKeags in #1438
-    - adds product key to error message in setup flow when there is mismatch by @dr-bonez in #1436
-    - Update README.md to include yq by @cryptodread in #1385
-    - yin_yang For the peace of mind yin_yang by @k0gen in #1444
-    - Feature/update sound by @ProofOfKeags in #1439
-    - Feature/script packing by @ProofOfKeags in #1435
-    - rename ActionImplementation to PackageProcedure by @dr-bonez in #1448
-    - Chore/warning cleanse by @ProofOfKeags in #1447
-    - refactor packing to async by @ProofOfKeags in #1453
-    - Add nginx config for proxy redirect by @yzernik in #1421
-    - Proxy local frontend to remote backend by @elvece in #1452
-    - Feat/js action by @Blu-J in #1437
-    - Fix/making js work by @Blu-J in #1456
-    - fix: Dependency vs dependents by @Blu-J in #1462
-    - refactor: isolate network toast and login redirect to separate services by @waterplea in #1412
-    - Fix links in CONTRIBUTING.md, update ToC by @BBlackwo in #1463
-    - Feature/require script consistency by @ProofOfKeags in #1451
-    - Chore/version 0 3 1 0 by @Blu-J in #1475
-    - remove interactive TTY requirement from scripts by @moerketh in #1469
-    - Disable view in marketplace button when side-loaded by @BBlackwo in #1471
-    - Link to tor address on LAN setup page (#1277) by @BBlackwo in #1466
-    - UI version updates and welcome message for 0.3.1 by @elvece in #1479
-    - Update contribution and frontend readme by @BBlackwo in #1467
-    - Clean up config by @MattDHill in #1484
-    - Enable Control Groups for Docker containers by @k0gen in #1468
-    - Fix/patch db unwrap remove by @Blu-J in #1481
-    - handles spaces in working dir in make-image.sh by @moerketh in #1487
-    - UI cosmetic improvements by @MattDHill in #1486
-    - chore: fix the master by @Blu-J in #1495
-    - generate unique ca names based off of server id by @ProofOfKeags in #1500
-    - allow embassy-cli not as root by @dr-bonez in #1501
-    - fix: potential fix for the docker leaking the errors and such by @Blu-J in #1496
-    - Fix/memory leak docker by @Blu-J in #1505
-    - fixes serialization of regex pattern + description by @ProofOfKeags in #1509
-    - allow interactive TTY if available by @dr-bonez in #1508
-    - fix "missing proxy" error in embassy-cli by @dr-bonez in #1516
-    - Feat/js known errors by @Blu-J in #1514
-    - fixes a bug where nginx will crash if eos goes into diagnostic mode a… by @dr-bonez in #1506
-    - fix: restart/ uninstall sometimes didn't work by @Blu-J in #1527
-    - add "error_for_status" to static file downloads by @dr-bonez in #1532
-    - fixes #1169 by @dr-bonez in #1533
-    - disable unnecessary services by @dr-bonez in #1535
-    - chore: Update types to match embassyd by @Blu-J in #1539
-    - fix: found a unsaturaded args fix by @Blu-J in #1540
-    - chore: Update the lite types to include the union and enum by @Blu-J in #1542
-    - Feat: Make the js check for health by @Blu-J in #1543
-    - fix incorrect error message for deserialization in ValueSpecString by @dr-bonez in #1547
-    - fix dependency/dependent id issue by @dr-bonez in #1546
-    - add textarea to ValueSpecString by @dr-bonez in #1534
-    - Feat/js metadata by @Blu-J in #1548
-    - feat: uid/gid/mode added to metadata by @Blu-J in #1551
-    - Strict null checks by @waterplea in #1464
-    - fix backend builds for safe git config by @elvece in #1549
-    - update should send version not version spec by @elvece in #1559
-    - chore: Add tracing for debuging the js procedure slowness by @Blu-J in #1552
-    - Reset password through setup wizard by @MattDHill in #1490
-    - feat: Make sdk by @Blu-J in #1564
-    - fix: Missing a feature flat cfg by @Blu-J in #1563
-    - fixed sentence that didn't make sense by @BitcoinMechanic in #1565
-    - refactor(patch-db): use PatchDB class declaratively by @waterplea in #1562
-    - fix bugs with config and clean up dev options by @MattDHill in #1558
-    - fix: Make it so we only need the password on the backup by @Blu-J in #1566
-    - kill all sessions and remove ripple effect by @MattDHill in #1567
-    - adjust service marketplace button for installation source relevance by @elvece in #1571
-    - fix connection failure display monitoring and other style changes by @MattDHill in #1573
-    - add dns server to embassy-os by @dr-bonez in #1572
-    - Fix/mask generic inputs by @elvece in #1570
-    - Fix/sideload icon type by @elvece in #1577
-    - add avahi conditional compilation flags to dns by @dr-bonez in #1579
-    - selective backups and better drive selection interface by @MattDHill in #1576
-    - Feat/use modern tor by @kn0wmad in #1575
-    - update welcome notes for 031 by @MattDHill in #1580
-    - fix: Properties had a null description by @Blu-J in #1581
-    - fix backup lock ordering by @dr-bonez in #1582
-    - Bugfix/backup lock order by @dr-bonez in #1583
-    - preload redacted and visibility hidden by @MattDHill in #1584
-    - turn chevron red in config if error by @MattDHill in #1586
-    - switch to utc by @dr-bonez in #1587
-    - update patchdb for array patch fix by @elvece in #1588
-    - filter package ids when backing up by @dr-bonez in #1589
-    - add select/deselect all to backups and enum lists by @elvece in #1590
-    - fix: Stop the buffer from dropped pre-maturly by @Blu-J in #1591
-    - chore: commit the snapshots by @Blu-J in #1592
-    - nest new entries and message updates better by @MattDHill in #1595
-    - fix html parsing in logs by @elvece in #1598
-    - don't crash service if io-format is set for main by @dr-bonez in #1599
-    - strip html from colors from logs by @elvece in #1604
-    - feat: fetch effect by @Blu-J in #1605
-    - Fix/UI misc by @elvece in #1606
-    - display bottom item in backup list and refactor for cleanliness by @MattDHill in #1609
-
-# v0.3.0.3
-## What's Changed
-	- refactor: decompose app component by @waterplea in #1359
-	- Update Makefile by @kn0wmad in #1400
-	- ⬐ smarter wget by @k0gen in #1401
-	- prevent the kernel from OOMKilling embassyd by @dr-bonez in #1402
-	- attempt to heal when health check passes by @dr-bonez in #1420
-	- Feat new locking by @Blu-J in #1384
-	- version bump by @dr-bonez in #1423
-	- Update server-show.page.ts by @chrisguida in #1424
-	- Bump async from 2.6.3 to 2.6.4 in /frontend by @dependabot in #1426
-	- Update index.html by @mirkoRainer in #1419
-
-## New Contributors
-    - @dependabot made their first contribution in #1426
-    - @mirkoRainer made their first contribution in #1419
-
-# v0.3.0.2
-- Minor compatibility fixes
-  - #1392
-  - #1390
-  - #1388
-
-# v0.3.0.1
-Minor bugfixes and performance improvements
-
-# v0.3.0
-- Websockets
-	- Real-time sync
-- Patch DB
-	- Closely mirror FE and BE state. Most operating systems are connected to their GUI. Here it is served over the web. Patch DB and websockets serve to close the perceptual gap of this inherent challenge.
-- Switch kernel from Raspbian to Ubuntu
-	- 64 bit
-	- Possibility for alternative hardware
-- Merging of lifeline, agent, and appmgr into embassyd
-	- Elimination of Haskell in favor of pure Rust
-	- Unified API for interacting with the OS
-	- Easier to build from source
-- OS (quarantined from OS and service data)
-	- Kernel/boot
-	- Persistent metadata (disk guid, product key)
-	- Rootfs (the os)
-	- Reserved (for updates) - swaps with rootfs
-- Revamped OS updates
-	- Progress indicators
-	- Non-blocking
-	- Simple swap on reboot
-- Revamped setup flow
-	- Elimination of Setup App (Apple/Google dependencies gone)
-	- Setup Wizard on http://embassy.local
-- Revamped service config
-	- Dynamic, validated forms
-- Diagnostic UI
-	- Missing disk, wrong disk, corrupt disk
-- Turing complete API for actions, backup/restore, config, properties, notifications, health checks, and dependency requirements
-- Optional, arbitrary inputs for actions
-- Install, update, recover progress for apps
-- Multiple interfaces
-	- E.g. rpc, p2p, ui
-- Health checks
-	- Developer defined
-	- Internal, dependencies, and/or external
-- Full Embassy backup (diff-based)
-- External drive support/requirement
-	- Single at first
-	- Groundwork for extension and mirror drives
-- Disk encryption
-	- Random key encrypted with static value
-	- Groundwork for swapping static value with chosen password
-- Session Management
-	- List all active sessions
-	- Option to kill
-- More robust and extensive logs
-- Donations

CONTRIBUTING.md

@@ -1,339 +1,119 @@
-<!-- omit in toc -->
-
 # Contributing to StartOS
 
-First off, thanks for taking the time to contribute! ❤️
+This guide is for contributing to the StartOS. If you are interested in packaging a service for StartOS, visit the [service packaging guide](https://docs.start9.com/latest/developer-docs/). If you are interested in promoting, providing technical support, creating tutorials, or helping in other ways, please visit the [Start9 website](https://start9.com/contribute).
 
-All types of contributions are encouraged and valued. See the
-[Table of Contents](#table-of-contents) for different ways to help and details
-about how this project handles them. Please make sure to read the relevant
-section before making your contribution. It will make it a lot easier for us
-maintainers and smooth out the experience for all involved. The community looks
-forward to your contributions. 🎉
 
-> And if you like the project, but just don't have time to contribute, that's
-> fine. There are other easy ways to support the project and show your
-> appreciation, which we would also be very happy about:
->
-> - Star the project
-> - Tweet about it
-> - Refer this project in your project's readme
-> - Mention the project at local meetups and tell your friends/colleagues
-> - Buy a [Start9 server](https://start9.com)
+## Collaboration
 
-<!-- omit in toc -->
+- [Matrix](https://matrix.to/#/#community-dev:matrix.start9labs.com)
+- [Telegram](https://t.me/start9_labs/47471)
 
-## Table of Contents
+## Project Structure
 
-- [I Have a Question](#i-have-a-question)
-- [I Want To Contribute](#i-want-to-contribute)
-  - [Reporting Bugs](#reporting-bugs)
-  - [Suggesting Enhancements](#suggesting-enhancements)
-  - [Project Structure](#project-structure)
-  - [Your First Code Contribution](#your-first-code-contribution)
-    - [Setting Up Your Development Environment](#setting-up-your-development-environment)
-    - [Building The Image](#building-the-image)
-  - [Improving The Documentation](#improving-the-documentation)
-- [Styleguides](#styleguides)
-  - [Formatting](#formatting)
-  - [Atomic Commits](#atomic-commits)
-  - [Commit Messages](#commit-messages)
-  - [Pull Requests](#pull-requests)
-  - [Rebasing Changes](#rebasing-changes)
-- [Join The Discussion](#join-the-discussion)
-- [Join The Project Team](#join-the-project-team)
+```bash
+/
+├── assets/
+├── core/
+├── build/
+├── debian/
+├── web/
+├── image-recipe/
+├── patch-db
+└── system-images/
+```
+#### assets
+screenshots for the StartOS README
 
-## I Have a Question
+#### core
+An API, daemon (startd), CLI (start-cli), and SDK (start-sdk) that together provide the core functionality of StartOS.
 
-> If you want to ask a question, we assume that you have read the available
-> [Documentation](https://docs.start9labs.com).
+#### build
+Auxiliary files and scripts to include in deployed StartOS images
 
-Before you ask a question, it is best to search for existing
-[Issues](https://github.com/Start9Labs/start-os/issues) that might help you.
-In case you have found a suitable issue and still need clarification, you can
-write your question in this issue. It is also advisable to search the internet
-for answers first.
+#### debian
+Maintainer scripts for the StartOS Debian package
 
-If you then still feel the need to ask a question and need clarification, we
-recommend the following:
+#### web
+Web UIs served under various conditions and used to interact with StartOS APIs.
 
-- Open an [Issue](https://github.com/Start9Labs/start-os/issues/new).
-- Provide as much context as you can about what you're running into.
-- Provide project and platform versions, depending on what seems relevant.
+#### image-recipe
+Scripts for building StartOS images
 
-We will then take care of the issue as soon as possible.
+#### patch-db (submodule)
+A diff based data store used to synchronize data between the web interfaces and server.
 
-<!--
-You might want to create a separate issue tag for questions and include it in this description. People should then tag their issues accordingly.
+#### system-images
+Docker images that assist with creating backups.
 
-Depending on how large the project is, you may want to outsource the questioning, e.g. to Stack Overflow or Gitter. You may add additional contact and information possibilities:
-- IRC
-- Slack
-- Gitter
-- Stack Overflow tag
-- Blog
-- FAQ
-- Roadmap
-- E-Mail List
-- Forum
--->
-
-## I Want To Contribute
-
-> ### Legal Notice <!-- omit in toc -->
->
-> When contributing to this project, you must agree that you have authored 100%
-> of the content, that you have the necessary rights to the content and that the
-> content you contribute may be provided under the project license.
-
-### Reporting Bugs
-
-<!-- omit in toc -->
-
-#### Before Submitting a Bug Report
-
-A good bug report shouldn't leave others needing to chase you up for more
-information. Therefore, we ask you to investigate carefully, collect information
-and describe the issue in detail in your report. Please complete the following
-steps in advance to help us fix any potential bug as fast as possible.
-
-- Make sure that you are using the latest version.
-- Determine if your bug is really a bug and not an error on your side e.g. using
-  incompatible environment components/versions (Make sure that you have read the
-  [documentation](https://start9.com/latest/user-manual). If you are looking for
-  support, you might want to check [this section](#i-have-a-question)).
-- To see if other users have experienced (and potentially already solved) the
-  same issue you are having, check if there is not already a bug report existing
-  for your bug or error in the
-  [bug tracker](https://github.com/Start9Labs/start-os/issues?q=label%3Abug).
-- Also make sure to search the internet (including Stack Overflow) to see if
-  users outside of the GitHub community have discussed the issue.
-- Collect information about the bug:
-  - Stack trace (Traceback)
-  - Client OS, Platform and Version (Windows/Linux/macOS/iOS/Android,
-    Firefox/Tor Browser/Consulate)
-  - Version of the interpreter, compiler, SDK, runtime environment, package
-    manager, depending on what seems relevant.
-  - Possibly your input and the output
-  - Can you reliably reproduce the issue? And can you also reproduce it with
-    older versions?
-
-<!-- omit in toc -->
-
-#### How Do I Submit a Good Bug Report?
-
-> You must never report security related issues, vulnerabilities or bugs to the
-> issue tracker, or elsewhere in public. Instead sensitive bugs must be sent by
-> email to <security@start9labs.com>.
-
-<!-- You may add a PGP key to allow the messages to be sent encrypted as well. -->
-
-We use GitHub issues to track bugs and errors. If you run into an issue with the
-project:
-
-- Open an [Issue](https://github.com/Start9Labs/start-os/issues/new/choose)
-  selecting the appropriate type.
-- Explain the behavior you would expect and the actual behavior.
-- Please provide as much context as possible and describe the _reproduction
-  steps_ that someone else can follow to recreate the issue on their own. This
-  usually includes your code. For good bug reports you should isolate the
-  problem and create a reduced test case.
-- Provide the information you collected in the previous section.
-
-Once it's filed:
-
-- The project team will label the issue accordingly.
-- A team member will try to reproduce the issue with your provided steps. If
-  there are no reproduction steps or no obvious way to reproduce the issue, the
-  team will ask you for those steps and mark the issue as `Question`. Bugs with
-  the `Question` tag will not be addressed until they are answered.
-- If the team is able to reproduce the issue, it will be marked a scoping level
-  tag, as well as possibly other tags (such as `Security`), and the issue will
-  be left to be [implemented by someone](#your-first-code-contribution).
-
-<!-- You might want to create an issue template for bugs and errors that can be used as a guide and that defines the structure of the information to be included. If you do so, reference it here in the description. -->
-
-### Suggesting Enhancements
-
-This section guides you through submitting an enhancement suggestion for StartOS, **including completely new features and minor improvements to existing
-functionality**. Following these guidelines will help maintainers and the
-community to understand your suggestion and find related suggestions.
-
-<!-- omit in toc -->
-
-#### Before Submitting an Enhancement
-
-- Make sure that you are using the latest version.
-- Read the [documentation](https://start9.com/latest/user-manual) carefully and
-  find out if the functionality is already covered, maybe by an individual
-  configuration.
-- Perform a [search](https://github.com/Start9Labs/start-os/issues) to see if
-  the enhancement has already been suggested. If it has, add a comment to the
-  existing issue instead of opening a new one.
-- Find out whether your idea fits with the scope and aims of the project. It's
-  up to you to make a strong case to convince the project's developers of the
-  merits of this feature. Keep in mind that we want features that will be useful
-  to the majority of our users and not just a small subset. If you're just
-  targeting a minority of users, consider writing an add-on/plugin library.
-
-<!-- omit in toc -->
-
-#### How Do I Submit a Good Enhancement Suggestion?
-
-Enhancement suggestions are tracked as
-[GitHub issues](https://github.com/Start9Labs/start-os/issues).
-
-- Use a **clear and descriptive title** for the issue to identify the
-  suggestion.
-- Provide a **step-by-step description of the suggested enhancement** in as many
-  details as possible.
-- **Describe the current behavior** and **explain which behavior you expected to
-  see instead** and why. At this point you can also tell which alternatives do
-  not work for you.
-- You may want to **include screenshots and animated GIFs** which help you
-  demonstrate the steps or point out the part which the suggestion is related
-  to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on
-  macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast)
-  or [this tool](https://github.com/GNOME/byzanz) on Linux.
-  <!-- this should only be included if the project has a GUI -->
-- **Explain why this enhancement would be useful** to most StartOS users. You
-  may also want to point out the other projects that solved it better and which
-  could serve as inspiration.
-
-<!-- You might want to create an issue template for enhancement suggestions that can be used as a guide and that defines the structure of the information to be included. If you do so, reference it here in the description. -->
-
-### Project Structure
-
-StartOS is composed of the following components. Please visit the README for
-each component to understand the dependency requirements and installation
-instructions.
-
-- [`backend`](backend/README.md) (Rust) is a command line utility, daemon, and
-  software development kit that sets up and manages services and their
-  environments, provides the interface for the ui, manages system state, and
-  provides utilities for packaging services for StartOS.
-- [`build`](build/README.md) contains scripts and necessary for deploying
-  StartOS to a debian/raspbian system.
-- [`frontend`](frontend/README.md) (Typescript Ionic Angular) is the code that
-  is deployed to the browser to provide the user interface for StartOS.
-  - `projects/ui` - Code for the user interface that is displayed when StartOS
-    is running normally.
-  - `projects/setup-wizard`(frontend/README.md) - Code for the user interface
-    that is displayed during the setup and recovery process for StartOS.
-  - `projects/diagnostic-ui` - Code for the user interface that is displayed
-    when something has gone wrong with starting up StartOS, which provides
-    helpful debugging tools.
-- `libs` (Rust) is a set of standalone crates that were separated out of
-  `backend` for the purpose of portability
-- `patch-db` - A diff based data store that is used to synchronize data between
-  the front and backend.
-  - Notably, `patch-db` has a
-    [client](https://github.com/Start9Labs/patch-db/tree/master/client) with its
-    own dependency and installation requirements.
-- `system-images` - (Docker, Rust) A suite of utility Docker images that are
-  preloaded with StartOS to assist with functions relating to services (eg.
-  configuration, backups, health checks).
-
-### Your First Code Contribution
-
-#### Setting Up Your Development Environment
-
-First, clone the StartOS repository and from the project root, pull in the
-submodules for dependent libraries.
+## Environment Setup
 
+#### Clone the StartOS repository
 ```sh
 git clone https://github.com/Start9Labs/start-os.git
+cd start-os
+```
+
+#### Load the PatchDB submodule
+```sh
 git submodule update --init --recursive
 ```
 
-Depending on which component of the ecosystem you are interested in contributing
-to, follow the installation requirements listed in that component's README
-(linked [above](#project-structure))
+#### Continue to your project of interest for additional instructions:
+- [`core`](core/README.md)
+- [`web-interfaces`](web-interfaces/README.md)
+- [`build`](build/README.md)
+- [`patch-db`](https://github.com/Start9Labs/patch-db)
 
-#### Building The Raspberry Pi Image
+## Building
+This project uses [GNU Make](https://www.gnu.org/software/make/) to build its components. To build any specific component, simply run `make <TARGET>` replacing `<TARGET>` with the name of the target you'd like to build
 
-This step is for setting up an environment in which to test your code changes if
-you do not yet have a StartOS.
+### Requirements
+- [GNU Make](https://www.gnu.org/software/make/)
+- [Docker](https://docs.docker.com/get-docker/)
+- [NodeJS v18.15.0](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm)
+- [sed](https://www.gnu.org/software/sed/)
+- [grep](https://www.gnu.org/software/grep/)
+- [awk](https://www.gnu.org/software/gawk/)
+- [jq](https://jqlang.github.io/jq/)
+- [gzip](https://www.gnu.org/software/gzip/)
+- [brotli](https://github.com/google/brotli)
 
-- Requirements
-  - `ext4fs` (available if running on the Linux kernel)
-  - [Docker](https://docs.docker.com/get-docker/)
-  - GNU Make
-- Building
-  - see setup instructions [here](build/README.md)
-  - run `make startos-raspi.img ARCH=aarch64` from the project root
+### Environment variables
+- `PLATFORM`: which platform you would like to build for. Must be one of `x86_64`, `x86_64-nonfree`, `aarch64`, `aarch64-nonfree`, `raspberrypi`
+    - NOTE: `nonfree` images are for including `nonfree` firmware packages in the built ISO
+- `ENVIRONMENT`: a hyphen separated set of feature flags to enable
+    - `dev`: enables password ssh (INSECURE!) and does not compress frontends
+    - `unstable`: enables assertions that will cause errors on unexpected inconsistencies that are undesirable in production use either for performance or reliability reasons
+    - `docker`: use `docker` instead of `podman`
+- `GIT_BRANCH_AS_HASH`: set to `1` to use the current git branch name as the git hash so that the project does not need to be rebuilt on each commit
 
-### Improving The Documentation
-
-You can find the repository for Start9's documentation
-[here](https://github.com/Start9Labs/documentation). If there is something you
-would like to see added, let us know, or create an issue yourself. Welcome are
-contributions for lacking or incorrect information, broken links, requested
-additions, or general style improvements.
-
-Contributions in the form of setup guides for integrations with external
-applications are highly encouraged. If you struggled through a process and would
-like to share your steps with others, check out the docs for each
-[service](https://github.com/Start9Labs/documentation/blob/master/source/user-manuals/available-services/index.rst)
-we support. The wrapper repos contain sections for adding integration guides,
-such as this
-[one](https://github.com/Start9Labs/bitcoind-wrapper/tree/master/docs). These
-not only help out others in the community, but inform how we can create a more
-seamless and intuitive experience.
-
-## Styleguides
-
-### Formatting
-
-Each component of StartOS contains its own style guide. Code must be formatted
-with the formatter designated for each component. These are outlined within each
-component folder's README.
-
-### Atomic Commits
-
-Commits
-[should be atomic](https://en.wikipedia.org/wiki/Atomic_commit#Atomic_commit_convention)
-and diffs should be easy to read. Do not mix any formatting fixes or code moves
-with actual code changes.
-
-### Commit Messages
-
-If a commit touches only 1 component, prefix the message with the affected
-component. i.e. `backend: update to tokio v0.3`.
-
-### Pull Requests
-
-The body of a pull request should contain sufficient description of what the
-changes do, as well as a justification. You should include references to any
-relevant [issues](https://github.com/Start9Labs/start-os/issues).
-
-### Rebasing Changes
-
-When a pull request conflicts with the target branch, you may be asked to rebase
-it on top of the current target branch. The `git rebase` command will take care
-of rebuilding your commits on top of the new base.
-
-This project aims to have a clean git history, where code changes are only made
-in non-merge commits. This simplifies auditability because merge commits can be
-assumed to not contain arbitrary code changes.
-
-## Join The Discussion
-
-Current or aspiring contributors? Join our community developer
-[Matrix channel](https://matrix.to/#/#community-dev:matrix.start9labs.com).
-
-Just interested in or using the project? Join our community
-[Telegram](https://t.me/start9_labs) or
-[Matrix](https://matrix.to/#/#community:matrix.start9labs.com).
-
-## Join The Project Team
-
-Interested in becoming a part of the Start9 Labs team? Send an email to
-<jobs@start9labs.com>
-
-<!-- omit in toc -->
-
-## Attribution
-
-This guide is based on the **contributing-gen**.
-[Make your own](https://github.com/bttger/contributing-gen)!
+### Useful Make Targets
+- `iso`: Create a full `.iso` image
+    - Only possible from Debian
+    - Not available for `PLATFORM=raspberrypi`
+    - Additional Requirements:
+        - [debspawn](https://github.com/lkhq/debspawn)
+- `img`: Create a full `.img` image
+    - Only possible from Debian
+    - Only available for `PLATFORM=raspberrypi`
+    - Additional Requirements:
+        - [debspawn](https://github.com/lkhq/debspawn)
+- `format`: Run automatic code formatting for the project
+    - Additional Requirements:
+        - [rust](https://rustup.rs/)
+- `test`: Run automated tests for the project
+    - Additional Requirements:
+        - [rust](https://rustup.rs/)
+- `update`: Deploy the current working project to a device over ssh as if through an over-the-air update
+    - Requires an argument `REMOTE` which is the ssh address of the device, i.e. `start9@192.168.122.2`
+- `reflash`: Deploy the current working project to a device over ssh as if using a live `iso` image to reflash it
+    - Requires an argument `REMOTE` which is the ssh address of the device, i.e. `start9@192.168.122.2`
+- `update-overlay`: Deploy the current working project to a device over ssh to the in-memory overlay without restarting it
+    - WARNING: changes will be reverted after the device is rebooted
+    - WARNING: changes to `init` will not take effect as the device is already initialized
+    - Requires an argument `REMOTE` which is the ssh address of the device, i.e. `start9@192.168.122.2`
+- `wormhole`: Deploy the `startbox` to a device using [magic-wormhole](https://github.com/magic-wormhole/magic-wormhole)
+    - When the build it complete will emit a command to paste into the shell of the device to upgrade it
+    - Additional Requirements:
+        - [magic-wormhole](https://github.com/magic-wormhole/magic-wormhole)
+- `clean`: Delete all compiled artifacts

DEVELOPMENT.md

@@ -0,0 +1,134 @@
+# Setting up your development environment on Debian/Ubuntu
+
+A step-by-step guide
+
+> This is the only officially supported build environment.
+> MacOS has limited build capabilities and Windows requires [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install)
+
+## Installing dependencies
+
+Run the following commands one at a time
+
+```sh
+sudo apt update
+sudo apt install -y ca-certificates curl gpg build-essential
+curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+echo "deb [arch=$(dpkg-architecture -q DEB_HOST_ARCH) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bookworm stable" | sudo tee /etc/apt/sources.list.d/docker.list
+sudo apt update
+sudo apt install -y sed grep gawk jq gzip brotli containerd.io docker-ce docker-ce-cli docker-compose-plugin qemu-user-static binfmt-support squashfs-tools git debspawn rsync b3sum
+sudo mkdir -p /etc/debspawn/
+echo "AllowUnsafePermissions=true" | sudo tee /etc/debspawn/global.toml
+sudo usermod -aG docker $USER
+sudo su $USER
+docker run --privileged --rm tonistiigi/binfmt --install all
+docker buildx create --use
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh # proceed with default installation
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/master/install.sh | bash
+source ~/.bashrc
+nvm install 20
+nvm use 20
+nvm alias default 20 # this prevents your machine from reverting back to another version
+```
+
+## Cloning the repository
+
+```sh
+git clone --recursive https://github.com/Start9Labs/start-os.git --branch next/minor
+cd start-os
+```
+
+## Building an ISO
+
+```sh
+PLATFORM=$(uname -m) ENVIRONMENT=dev make iso
+```
+
+This will build an ISO for your current architecture. If you are building to run on an architecture other than the one you are currently on, replace `$(uname -m)` with the correct platform for the device (one of `aarch64`, `aarch64-nonfree`, `x86_64`, `x86_64-nonfree`, `raspberrypi`)
+
+## Creating a VM
+
+### Install virt-manager
+
+```sh
+sudo apt update
+sudo apt install -y virt-manager
+sudo usermod -aG libvirt $USER
+sudo su $USER
+```
+
+### Launch virt-manager
+
+```sh
+virt-manager
+```
+
+### Create new virtual machine
+
+![Select "Create a new virtual machine"](assets/create-vm/step-1.png)
+![Click "Forward"](assets/create-vm/step-2.png)
+![Click "Browse"](assets/create-vm/step-3.png)
+![Click "+"](assets/create-vm/step-4.png)
+
+#### make sure to set "Target Path" to the path to your results directory in start-os
+
+![Create storage pool](assets/create-vm/step-5.png)
+![Select storage pool](assets/create-vm/step-6.png)
+![Select ISO](assets/create-vm/step-7.png)
+![Select "Generic or unknown OS" and click "Forward"](assets/create-vm/step-8.png)
+![Set Memory and CPUs](assets/create-vm/step-9.png)
+![Create disk](assets/create-vm/step-10.png)
+![Name VM](assets/create-vm/step-11.png)
+![Create network](assets/create-vm/step-12.png)
+
+## Updating a VM
+
+The fastest way to update a VM to your latest code depends on what you changed:
+
+### UI or startd:
+
+```sh
+PLATFORM=$(uname -m) ENVIRONMENT=dev make update-startbox REMOTE=start9@<VM IP>
+```
+
+### Container runtime or debian dependencies:
+
+```sh
+PLATFORM=$(uname -m) ENVIRONMENT=dev make update-deb REMOTE=start9@<VM IP>
+```
+
+### Image recipe:
+
+```sh
+PLATFORM=$(uname -m) ENVIRONMENT=dev make update-squashfs REMOTE=start9@<VM IP>
+```
+
+---
+
+If the device you are building for is not available via ssh, it is also possible to use `magic-wormhole` to send the relevant files.
+
+### Prerequisites:
+
+```sh
+sudo apt update
+sudo apt install -y magic-wormhole
+```
+
+As before, the fastest way to update a VM to your latest code depends on what you changed. Each of the following commands will return a command to paste into the shell of the device you would like to upgrade.
+
+### UI or startd:
+
+```sh
+PLATFORM=$(uname -m) ENVIRONMENT=dev make wormhole
+```
+
+### Container runtime or debian dependencies:
+
+```sh
+PLATFORM=$(uname -m) ENVIRONMENT=dev make wormhole-deb
+```
+
+### Image recipe:
+
+```sh
+PLATFORM=$(uname -m) ENVIRONMENT=dev make wormhole-squashfs
+```

Makefile

@@ -6,26 +6,27 @@ BASENAME := $(shell ./basename.sh)
 PLATFORM := $(shell if [ -f ./PLATFORM.txt ]; then cat ./PLATFORM.txt; else echo unknown; fi)
 ARCH := $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo aarch64; else echo $(PLATFORM) | sed 's/-nonfree$$//g'; fi)
 IMAGE_TYPE=$(shell if [ "$(PLATFORM)" = raspberrypi ]; then echo img; else echo iso; fi)
-EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/startbox libs/target/aarch64-unknown-linux-musl/release/embassy_container_init libs/target/x86_64-unknown-linux-musl/release/embassy_container_init
-EMBASSY_UIS := frontend/dist/raw/ui frontend/dist/raw/setup-wizard frontend/dist/raw/diagnostic-ui frontend/dist/raw/install-wizard
-BUILD_SRC := $(shell git ls-files build) build/lib/depends build/lib/conflicts
+WEB_UIS := web/dist/raw/ui/index.html web/dist/raw/setup-wizard/index.html web/dist/raw/install-wizard/index.html
+COMPRESSED_WEB_UIS := web/dist/static/ui/index.html web/dist/static/setup-wizard/index.html web/dist/static/install-wizard/index.html
+FIRMWARE_ROMS := ./firmware/$(PLATFORM) $(shell jq --raw-output '.[] | select(.platform[] | contains("$(PLATFORM)")) | "./firmware/$(PLATFORM)/" + .id + ".rom.gz"' build/lib/firmware.json)
+BUILD_SRC := $(shell git ls-files build) build/lib/depends build/lib/conflicts $(FIRMWARE_ROMS)
 DEBIAN_SRC := $(shell git ls-files debian/)
 IMAGE_RECIPE_SRC := $(shell git ls-files image-recipe/)
-EMBASSY_SRC := backend/startd.service $(BUILD_SRC)
+STARTD_SRC := core/startos/startd.service $(BUILD_SRC)
 COMPAT_SRC := $(shell git ls-files system-images/compat/)
 UTILS_SRC := $(shell git ls-files system-images/utils/)
 BINFMT_SRC := $(shell git ls-files system-images/binfmt/)
-BACKEND_SRC := $(shell git ls-files backend) $(shell git ls-files --recurse-submodules patch-db) $(shell git ls-files libs) frontend/dist/static
-FRONTEND_SHARED_SRC := $(shell git ls-files frontend/projects/shared) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json
-FRONTEND_UI_SRC := $(shell git ls-files frontend/projects/ui)
-FRONTEND_SETUP_WIZARD_SRC := $(shell git ls-files frontend/projects/setup-wizard)
-FRONTEND_DIAGNOSTIC_UI_SRC := $(shell git ls-files frontend/projects/diagnostic-ui)
-FRONTEND_INSTALL_WIZARD_SRC := $(shell git ls-files frontend/projects/install-wizard)
+CORE_SRC := $(shell git ls-files core) $(shell git ls-files --recurse-submodules patch-db) $(GIT_HASH_FILE)
+WEB_SHARED_SRC := $(shell git ls-files web/projects/shared) $(shell ls -p web/ | grep -v / | sed 's/^/web\//g') web/node_modules/.package-lock.json web/config.json patch-db/client/dist/index.js sdk/baseDist/package.json web/patchdb-ui-seed.json sdk/dist/package.json
+WEB_UI_SRC := $(shell git ls-files web/projects/ui)
+WEB_SETUP_WIZARD_SRC := $(shell git ls-files web/projects/setup-wizard)
+WEB_INSTALL_WIZARD_SRC := $(shell git ls-files web/projects/install-wizard)
 PATCH_DB_CLIENT_SRC := $(shell git ls-files --recurse-submodules patch-db/client)
 GZIP_BIN := $(shell which pigz || which gzip)
 TAR_BIN := $(shell which gtar || which tar)
-COMPILED_TARGETS := $(EMBASSY_BINS) system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar
-ALL_TARGETS := $(EMBASSY_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE) $(COMPILED_TARGETS) $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep; fi)  $(shell /bin/bash -c 'if [[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]; then echo cargo-deps/$(ARCH)-unknown-linux-gnu/release/tokio-console; fi') $(PLATFORM_FILE)
+COMPILED_TARGETS := core/target/$(ARCH)-unknown-linux-musl/release/startbox core/target/$(ARCH)-unknown-linux-musl/release/containerbox system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar container-runtime/rootfs.$(ARCH).squashfs
+ALL_TARGETS := $(STARTD_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE) $(COMPILED_TARGETS) cargo-deps/$(ARCH)-unknown-linux-musl/release/startos-backup-fs $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-musl/release/pi-beep; fi)  $(shell /bin/bash -c 'if [[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]; then echo cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console; fi') $(PLATFORM_FILE) 
+REBUILD_TYPES = 1
 
 ifeq ($(REMOTE),)
 	mkdir = mkdir -p $1
@@ -48,10 +49,13 @@ endif
 
 .DELETE_ON_ERROR:
 
-.PHONY: all metadata install clean format sdk snapshots frontends ui backend reflash deb $(IMAGE_TYPE) squashfs sudo wormhole docker-buildx
+.PHONY: all metadata install clean format cli uis ui reflash deb $(IMAGE_TYPE) squashfs sudo wormhole wormhole-deb test test-core test-sdk test-container-runtime registry
 
 all: $(ALL_TARGETS)
 
+touch:
+	touch $(ALL_TARGETS)
+
 metadata: $(VERSION_FILE) $(PLATFORM_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE)
 
 sudo:
@@ -60,12 +64,12 @@ sudo:
 clean:
 	rm -f system-images/**/*.tar
 	rm -rf system-images/compat/target
-	rm -rf backend/target
-	rm -rf frontend/.angular
-	rm -f frontend/config.json
-	rm -rf frontend/node_modules
-	rm -rf frontend/dist
-	rm -rf libs/target
+	rm -rf core/target
+	rm -rf core/startos/bindings
+	rm -rf web/.angular
+	rm -f web/config.json
+	rm -rf web/node_modules
+	rm -rf web/dist
 	rm -rf patch-db/client/node_modules
 	rm -rf patch-db/client/dist
 	rm -rf patch-db/target
@@ -73,24 +77,45 @@ clean:
 	rm -rf dpkg-workdir
 	rm -rf image-recipe/deb
 	rm -rf results
+	rm -rf build/lib/firmware
+	rm -rf container-runtime/dist
+	rm -rf container-runtime/node_modules
+	rm -f container-runtime/*.squashfs
+	if [ -d container-runtime/tmp/combined ] && mountpoint container-runtime/tmp/combined; then sudo umount container-runtime/tmp/combined; fi
+	if [ -d container-runtime/tmp/lower ] && mountpoint container-runtime/tmp/lower; then sudo umount container-runtime/tmp/lower; fi
+	rm -rf container-runtime/tmp
+	(cd sdk && make clean)
 	rm -f ENVIRONMENT.txt
 	rm -f PLATFORM.txt
 	rm -f GIT_HASH.txt
 	rm -f VERSION.txt
 
 format:
-	cd backend && cargo +nightly fmt
-	cd libs && cargo +nightly fmt
+	cd core && cargo +nightly fmt
 
-sdk:
-	cd backend/ && ./install-sdk.sh
+test: | test-core test-sdk test-container-runtime
+
+test-core: $(CORE_SRC) $(ENVIRONMENT_FILE) 
+	./core/run-tests.sh
+
+test-sdk: $(shell git ls-files sdk) sdk/base/lib/osBindings/index.ts
+	cd sdk && make test
+
+test-container-runtime: container-runtime/node_modules/.package-lock.json $(shell git ls-files container-runtime/src) container-runtime/package.json container-runtime/tsconfig.json 
+	cd container-runtime && npm test
+
+cli:
+	cd core && ./install-cli.sh
+
+registry:
+	cd core && ./build-registrybox.sh
 
 deb: results/$(BASENAME).deb
 
 debian/control: build/lib/depends build/lib/conflicts
 	./debuild/control.sh
 
-results/$(BASENAME).deb: dpkg-build.sh $(DEBIAN_SRC) $(VERSION_FILE) $(PLATFORM_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE)
+results/$(BASENAME).deb: dpkg-build.sh $(DEBIAN_SRC) $(ALL_TARGETS)
 	PLATFORM=$(PLATFORM) ./dpkg-build.sh
 
 $(IMAGE_TYPE): results/$(BASENAME).$(IMAGE_TYPE)
@@ -101,38 +126,37 @@ results/$(BASENAME).$(IMAGE_TYPE) results/$(BASENAME).squashfs: $(IMAGE_RECIPE_S
 	./image-recipe/run-local-build.sh "results/$(BASENAME).deb"
 
 # For creating os images. DO NOT USE
-install: $(ALL_TARGETS)
+install: $(ALL_TARGETS) 
 	$(call mkdir,$(DESTDIR)/usr/bin)
-	$(call cp,backend/target/$(ARCH)-unknown-linux-gnu/release/startbox,$(DESTDIR)/usr/bin/startbox)
+	$(call mkdir,$(DESTDIR)/usr/sbin)
+	$(call cp,core/target/$(ARCH)-unknown-linux-musl/release/startbox,$(DESTDIR)/usr/bin/startbox)
 	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/startd)
 	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/start-cli)
 	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/start-sdk)
-	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/start-deno)
-	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/avahi-alias)
-	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/embassy-cli)
-	if [ "$(PLATFORM)" = "raspberrypi" ]; then $(call cp,cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep,$(DESTDIR)/usr/bin/pi-beep); fi
-	if /bin/bash -c '[[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]'; then $(call cp,cargo-deps/$(ARCH)-unknown-linux-gnu/release/tokio-console,$(DESTDIR)/usr/bin/tokio-console); fi
+	if [ "$(PLATFORM)" = "raspberrypi" ]; then $(call cp,cargo-deps/aarch64-unknown-linux-musl/release/pi-beep,$(DESTDIR)/usr/bin/pi-beep); fi
+	if /bin/bash -c '[[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]'; then $(call cp,cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console,$(DESTDIR)/usr/bin/tokio-console); fi
+	$(call cp,cargo-deps/$(ARCH)-unknown-linux-musl/release/startos-backup-fs,$(DESTDIR)/usr/bin/startos-backup-fs)
+	$(call ln,/usr/bin/startos-backup-fs,$(DESTDIR)/usr/sbin/mount.backup-fs)
 	
 	$(call mkdir,$(DESTDIR)/lib/systemd/system)
-	$(call cp,backend/startd.service,$(DESTDIR)/lib/systemd/system/startd.service)
+	$(call cp,core/startos/startd.service,$(DESTDIR)/lib/systemd/system/startd.service)
 
 	$(call mkdir,$(DESTDIR)/usr/lib)
 	$(call rm,$(DESTDIR)/usr/lib/startos)
 	$(call cp,build/lib,$(DESTDIR)/usr/lib/startos)
+	$(call mkdir,$(DESTDIR)/usr/lib/startos/container-runtime)
+	$(call cp,container-runtime/rootfs.$(ARCH).squashfs,$(DESTDIR)/usr/lib/startos/container-runtime/rootfs.squashfs)
 
 	$(call cp,PLATFORM.txt,$(DESTDIR)/usr/lib/startos/PLATFORM.txt)
 	$(call cp,ENVIRONMENT.txt,$(DESTDIR)/usr/lib/startos/ENVIRONMENT.txt)
 	$(call cp,GIT_HASH.txt,$(DESTDIR)/usr/lib/startos/GIT_HASH.txt)
 	$(call cp,VERSION.txt,$(DESTDIR)/usr/lib/startos/VERSION.txt)
 
-	$(call mkdir,$(DESTDIR)/usr/lib/startos/container)
-	$(call cp,libs/target/aarch64-unknown-linux-musl/release/embassy_container_init,$(DESTDIR)/usr/lib/startos/container/embassy_container_init.arm64)
-	$(call cp,libs/target/x86_64-unknown-linux-musl/release/embassy_container_init,$(DESTDIR)/usr/lib/startos/container/embassy_container_init.amd64)
-
 	$(call mkdir,$(DESTDIR)/usr/lib/startos/system-images)
 	$(call cp,system-images/compat/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/compat.tar)
 	$(call cp,system-images/utils/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/utils.tar)
-	$(call cp,system-images/binfmt/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/binfmt.tar)
+	
+	$(call cp,firmware/$(PLATFORM),$(DESTDIR)/usr/lib/startos/firmware)
 
 update-overlay: $(ALL_TARGETS)
 	@echo "\033[33m!!! THIS WILL ONLY REFLASH YOUR DEVICE IN MEMORY !!!\033[0m"
@@ -143,88 +167,170 @@ update-overlay: $(ALL_TARGETS)
 	$(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) PLATFORM=$(PLATFORM)
 	$(call ssh,"sudo systemctl start startd")
 
-wormhole: backend/target/$(ARCH)-unknown-linux-gnu/release/startbox
-	@wormhole send backend/target/$(ARCH)-unknown-linux-gnu/release/startbox 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade \"cd /usr/bin && rm startbox && wormhole receive --accept-file %s && chmod +x startbox\"\n", $$3 }'
+wormhole: core/target/$(ARCH)-unknown-linux-musl/release/startbox
+	@echo "Paste the following command into the shell of your StartOS server:"
+	@echo
+	@wormhole send core/target/$(ARCH)-unknown-linux-musl/release/startbox 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade \"cd /usr/bin && rm startbox && wormhole receive --accept-file %s && chmod +x startbox\"\n", $$3 }'
+
+wormhole-deb: results/$(BASENAME).deb
+	@echo "Paste the following command into the shell of your StartOS server:"
+	@echo
+	@wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }'
+
+wormhole-squashfs: results/$(BASENAME).squashfs
+	$(eval SQFS_SUM := $(shell b3sum results/$(BASENAME).squashfs | head -c 32))
+	$(eval SQFS_SIZE := $(shell du -s --bytes results/$(BASENAME).squashfs | awk '{print $$1}'))
+	@echo "Paste the following command into the shell of your StartOS server:"
+	@echo
+	@wormhole send results/$(BASENAME).squashfs 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo sh -c '"'"'/usr/lib/startos/scripts/prune-images $(SQFS_SIZE) && cd /media/startos/images && wormhole receive --accept-file %s && mv $(BASENAME).squashfs $(SQFS_SUM).rootfs && ln -rsf ./$(SQFS_SUM).rootfs ../config/current.rootfs && sync && reboot'"'"'\n", $$3 }'
 
 update: $(ALL_TARGETS)
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
-	$(call ssh,"sudo rsync -a --delete --force --info=progress2 /media/embassy/embassyfs/current/ /media/embassy/next/")
-	$(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) DESTDIR=/media/embassy/next PLATFORM=$(PLATFORM)
-	$(call ssh,'sudo NO_SYNC=1 /media/embassy/next/usr/lib/startos/scripts/chroot-and-upgrade "apt-get install -y $(shell cat ./build/lib/depends)"')
+	$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')
+	$(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) DESTDIR=/media/startos/next PLATFORM=$(PLATFORM)
+	$(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y $(shell cat ./build/lib/depends)"')
+
+update-startbox: core/target/$(ARCH)-unknown-linux-musl/release/startbox # only update binary (faster than full update)
+	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
+	$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')
+	$(call cp,core/target/$(ARCH)-unknown-linux-musl/release/startbox,/media/startos/next/usr/bin/startbox)
+	$(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync true')
+
+update-deb: results/$(BASENAME).deb # better than update, but only available from debian
+	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
+	$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')
+	$(call mkdir,/media/startos/next/tmp/startos-deb)
+	$(call cp,results/$(BASENAME).deb,/media/startos/next/tmp/startos-deb/$(BASENAME).deb)
+	$(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y --reinstall /tmp/startos-deb/$(BASENAME).deb"')
+
+update-squashfs: results/$(BASENAME).squashfs
+	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
+	$(eval SQFS_SUM := $(shell b3sum results/$(BASENAME).squashfs))
+	$(eval SQFS_SIZE := $(shell du -s --bytes results/$(BASENAME).squashfs | awk '{print $$1}'))
+	$(call ssh,'/usr/lib/startos/scripts/prune-images $(SQFS_SIZE)')
+	$(call cp,results/$(BASENAME).squashfs,/media/startos/images/$(SQFS_SUM).rootfs)
+	$(call ssh,'sudo ln -rsf /media/startos/images/$(SQFS_SUM).rootfs /media/startos/config/current.rootfs')
+	$(call ssh,'sudo reboot')
 
 emulate-reflash: $(ALL_TARGETS)
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
-	$(call ssh,"sudo rsync -a --delete --force --info=progress2 /media/embassy/embassyfs/current/ /media/embassy/next/")
-	$(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) DESTDIR=/media/embassy/next PLATFORM=$(PLATFORM)
-	$(call ssh,"sudo touch /media/embassy/config/upgrade && sudo rm -f /media/embassy/config/disk.guid && sudo sync && sudo reboot")
+	$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')
+	$(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) DESTDIR=/media/startos/next PLATFORM=$(PLATFORM)
+	$(call ssh,'sudo rm -f /media/startos/config/disk.guid /media/startos/config/overlay/etc/hostname')
+	$(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y $(shell cat ./build/lib/depends)"')
 
 upload-ota: results/$(BASENAME).squashfs
 	TARGET=$(TARGET) KEY=$(KEY) ./upload-ota.sh
 
+container-runtime/debian.$(ARCH).squashfs:
+	ARCH=$(ARCH) ./container-runtime/download-base-image.sh
+
+container-runtime/node_modules/.package-lock.json: container-runtime/package.json container-runtime/package-lock.json sdk/dist/package.json
+	npm --prefix container-runtime ci
+	touch container-runtime/node_modules/.package-lock.json
+
+sdk/base/lib/osBindings/index.ts: $(shell if [ "$(REBUILD_TYPES)" -ne 0 ]; then echo core/startos/bindings/index.ts; fi)
+	mkdir -p sdk/base/lib/osBindings
+	rsync -ac --delete core/startos/bindings/ sdk/base/lib/osBindings/
+	touch sdk/base/lib/osBindings/index.ts
+
+core/startos/bindings/index.ts: $(shell git ls-files core) $(ENVIRONMENT_FILE)
+	rm -rf core/startos/bindings
+	./core/build-ts.sh
+	ls core/startos/bindings/*.ts | sed 's/core\/startos\/bindings\/\([^.]*\)\.ts/export { \1 } from ".\/\1";/g' | grep -v '"./index"' | tee core/startos/bindings/index.ts
+	npm --prefix sdk exec -- prettier --config ./sdk/base/package.json -w ./core/startos/bindings/*.ts
+	touch core/startos/bindings/index.ts
+
+sdk/dist/package.json sdk/baseDist/package.json: $(shell git ls-files sdk) sdk/base/lib/osBindings/index.ts
+	(cd sdk && make bundle)
+	touch sdk/dist/package.json
+	touch sdk/baseDist/package.json
+
+# TODO: make container-runtime its own makefile?
+container-runtime/dist/index.js: container-runtime/node_modules/.package-lock.json $(shell git ls-files container-runtime/src) container-runtime/package.json container-runtime/tsconfig.json 
+	npm --prefix container-runtime run build
+
+container-runtime/dist/node_modules/.package-lock.json container-runtime/dist/package.json container-runtime/dist/package-lock.json: container-runtime/package.json container-runtime/package-lock.json sdk/dist/package.json container-runtime/install-dist-deps.sh
+	./container-runtime/install-dist-deps.sh
+	touch container-runtime/dist/node_modules/.package-lock.json
+
+container-runtime/rootfs.$(ARCH).squashfs: container-runtime/debian.$(ARCH).squashfs container-runtime/container-runtime.service container-runtime/update-image.sh container-runtime/deb-install.sh container-runtime/dist/index.js container-runtime/dist/node_modules/.package-lock.json core/target/$(ARCH)-unknown-linux-musl/release/containerbox | sudo
+	ARCH=$(ARCH) ./container-runtime/update-image.sh
+
 build/lib/depends build/lib/conflicts: build/dpkg-deps/*
 	build/dpkg-deps/generate.sh
 
-system-images/compat/docker-images/$(ARCH).tar: $(COMPAT_SRC) backend/Cargo.lock | docker-buildx
+$(FIRMWARE_ROMS): build/lib/firmware.json download-firmware.sh $(PLATFORM_FILE)
+	./download-firmware.sh $(PLATFORM)
+
+system-images/compat/docker-images/$(ARCH).tar: $(COMPAT_SRC)
 	cd system-images/compat && make docker-images/$(ARCH).tar && touch docker-images/$(ARCH).tar
 
-system-images/utils/docker-images/$(ARCH).tar: $(UTILS_SRC) | docker-buildx
+system-images/utils/docker-images/$(ARCH).tar: $(UTILS_SRC)
 	cd system-images/utils && make docker-images/$(ARCH).tar && touch docker-images/$(ARCH).tar
 
-system-images/binfmt/docker-images/$(ARCH).tar: $(BINFMT_SRC) | docker-buildx
+system-images/binfmt/docker-images/$(ARCH).tar: $(BINFMT_SRC)
 	cd system-images/binfmt && make docker-images/$(ARCH).tar && touch docker-images/$(ARCH).tar
 
-snapshots: libs/snapshot_creator/Cargo.toml
-	cd libs/  && ./build-v8-snapshot.sh
-	cd libs/  && ./build-arm-v8-snapshot.sh
+core/target/$(ARCH)-unknown-linux-musl/release/startbox: $(CORE_SRC) $(COMPRESSED_WEB_UIS) web/patchdb-ui-seed.json $(ENVIRONMENT_FILE)
+	ARCH=$(ARCH) ./core/build-startbox.sh
+	touch core/target/$(ARCH)-unknown-linux-musl/release/startbox
 
-$(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) frontend/patchdb-ui-seed.json
-	cd backend && ARCH=$(ARCH) ./build-prod.sh
-	touch $(EMBASSY_BINS)
+core/target/$(ARCH)-unknown-linux-musl/release/containerbox: $(CORE_SRC) $(ENVIRONMENT_FILE)
+	ARCH=$(ARCH) ./core/build-containerbox.sh
+	touch core/target/$(ARCH)-unknown-linux-musl/release/containerbox
 
-frontend/node_modules: frontend/package.json
-	npm --prefix frontend ci
+web/node_modules/.package-lock.json: web/package.json sdk/baseDist/package.json
+	npm --prefix web ci
+	touch web/node_modules/.package-lock.json
 
-frontend/dist/raw/ui: $(FRONTEND_UI_SRC) $(FRONTEND_SHARED_SRC)
-	npm --prefix frontend run build:ui
+web/.angular/.updated: patch-db/client/dist/index.js sdk/baseDist/package.json web/node_modules/.package-lock.json
+	rm -rf web/.angular
+	mkdir -p web/.angular
+	touch web/.angular/.updated
 
-frontend/dist/raw/setup-wizard: $(FRONTEND_SETUP_WIZARD_SRC) $(FRONTEND_SHARED_SRC)
-	npm --prefix frontend run build:setup
+web/dist/raw/ui/index.html: $(WEB_UI_SRC) $(WEB_SHARED_SRC) web/.angular/.updated
+	npm --prefix web run build:ui
+	touch web/dist/raw/ui/index.html
 
-frontend/dist/raw/diagnostic-ui: $(FRONTEND_DIAGNOSTIC_UI_SRC) $(FRONTEND_SHARED_SRC)
-	npm --prefix frontend run build:dui
+web/dist/raw/setup-wizard/index.html: $(WEB_SETUP_WIZARD_SRC) $(WEB_SHARED_SRC) web/.angular/.updated
+	npm --prefix web run build:setup
+	touch web/dist/raw/setup-wizard/index.html
 
-frontend/dist/raw/install-wizard: $(FRONTEND_INSTALL_WIZARD_SRC) $(FRONTEND_SHARED_SRC)
-	npm --prefix frontend run build:install-wiz
+web/dist/raw/install-wizard/index.html: $(WEB_INSTALL_WIZARD_SRC) $(WEB_SHARED_SRC) web/.angular/.updated
+	npm --prefix web run build:install-wiz
+	touch web/dist/raw/install-wizard/index.html
 
-frontend/dist/static: $(EMBASSY_UIS) $(ENVIRONMENT_FILE)
+$(COMPRESSED_WEB_UIS): $(WEB_UIS) $(ENVIRONMENT_FILE)
 	./compress-uis.sh
 
-frontend/config.json: $(GIT_HASH_FILE) frontend/config-sample.json
-	jq '.useMocks = false' frontend/config-sample.json | jq '.gitHash = "$(shell cat GIT_HASH.txt)"' > frontend/config.json
+web/config.json: $(GIT_HASH_FILE) web/config-sample.json
+	jq '.useMocks = false' web/config-sample.json | jq '.gitHash = "$(shell cat GIT_HASH.txt)"' > web/config.json
 
-frontend/patchdb-ui-seed.json: frontend/package.json
-	jq '."ack-welcome" = $(shell jq '.version' frontend/package.json)' frontend/patchdb-ui-seed.json > ui-seed.tmp
-	mv ui-seed.tmp frontend/patchdb-ui-seed.json
-
-patch-db/client/node_modules: patch-db/client/package.json
+patch-db/client/node_modules/.package-lock.json: patch-db/client/package.json
 	npm --prefix patch-db/client ci
+	touch patch-db/client/node_modules/.package-lock.json
 
-patch-db/client/dist: $(PATCH_DB_CLIENT_SRC) patch-db/client/node_modules
-	! test -d patch-db/client/dist || rm -rf patch-db/client/dist
-	npm --prefix frontend run build:deps
+patch-db/client/dist/index.js: $(PATCH_DB_CLIENT_SRC) patch-db/client/node_modules/.package-lock.json
+	rm -rf patch-db/client/dist
+	npm --prefix patch-db/client run build
+	touch patch-db/client/dist/index.js
 
 # used by github actions
 compiled-$(ARCH).tar: $(COMPILED_TARGETS) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE)
 	tar -cvf $@ $^
 
-# this is a convenience step to build all frontends - it is not referenced elsewhere in this file
-frontends: $(EMBASSY_UIS) 
+# this is a convenience step to build all web uis - it is not referenced elsewhere in this file
+uis: $(WEB_UIS) 
 
 # this is a convenience step to build the UI
-ui: frontend/dist/raw/ui
+ui: web/dist/raw/ui
 
-cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep:
+cargo-deps/aarch64-unknown-linux-musl/release/pi-beep:
 	ARCH=aarch64 ./build-cargo-dep.sh pi-beep
 
-cargo-deps/$(ARCH)-unknown-linux-gnu/release/tokio-console:
-	ARCH=$(ARCH) ./build-cargo-dep.sh tokio-console
+cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console:
+	ARCH=$(ARCH) PREINSTALL="apk add musl-dev pkgconfig" ./build-cargo-dep.sh tokio-console
+
+cargo-deps/$(ARCH)-unknown-linux-musl/release/startos-backup-fs:
+	ARCH=$(ARCH) PREINSTALL="apk add fuse3 fuse3-dev fuse3-static musl-dev pkgconfig" ./build-cargo-dep.sh --git https://github.com/Start9Labs/start-fs.git startos-backup-fs

README.md

@@ -1,5 +1,5 @@
 <div align="center">
-  <img src="frontend/projects/shared/assets/img/icon.png" alt="StartOS Logo" width="16%" />
+  <img src="web/projects/shared/assets/img/icon.png" alt="StartOS Logo" width="16%" />
   <h1 style="margin-top: 0;">StartOS</h1>
   <a href="https://github.com/Start9Labs/start-os/releases">
     <img alt="GitHub release (with filter)" src="https://img.shields.io/github/v/release/start9labs/start-os?logo=github">
@@ -48,7 +48,8 @@
 <br />
 
 ## Running StartOS
-There are multiple ways to get started with StartOS:
+> [!WARNING]
+> StartOS is in beta. It lacks features. It doesn't always work perfectly. Start9 servers are not plug and play. Using them properly requires some effort and patience. Please do not use StartOS or purchase a server if you are unable or unwilling to follow instructions and learn new concepts.
 
 ### 💰 Buy a Start9 server
 This is the most convenient option. Simply [buy a server](https://store.start9.com) from Start9 and plug it in.

backend/.gitignore

@@ -1,10 +0,0 @@
-/target
-**/*.rs.bk
-.DS_Store
-.vscode
-secrets.db
-*.s9pk
-*.sqlite3
-.env
-.editorconfig
-proptest-regressions/*

backend/.sqlx/query-1ce5254f27de971fd87f5ab66d300f2b22433c86617a0dbf796bf2170186dd2e.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Bytea"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "1ce5254f27de971fd87f5ab66d300f2b22433c86617a0dbf796bf2170186dd2e"
-}

backend/.sqlx/query-21471490cdc3adb206274cc68e1ea745ffa5da4479478c1fd2158a45324b1930.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM ssh_keys WHERE fingerprint = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "21471490cdc3adb206274cc68e1ea745ffa5da4479478c1fd2158a45324b1930"
-}

backend/.sqlx/query-28ea34bbde836e0618c5fc9bb7c36e463c20c841a7d6a0eb15be0f24f4a928ec.json

@@ -1,40 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT hostname, path, username, password FROM cifs_shares WHERE id = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "hostname",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 1,
-        "name": "path",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "username",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 3,
-        "name": "password",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      true
-    ]
-  },
-  "hash": "28ea34bbde836e0618c5fc9bb7c36e463c20c841a7d6a0eb15be0f24f4a928ec"
-}

backend/.sqlx/query-4099028a5c0de578255bf54a67cef6cb0f1e9a4e158260700f1639dd4b438997.json

@@ -1,34 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT * FROM ssh_keys WHERE fingerprint = $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "fingerprint",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 1,
-        "name": "openssh_pubkey",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "created_at",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "4099028a5c0de578255bf54a67cef6cb0f1e9a4e158260700f1639dd4b438997"
-}

backend/.sqlx/query-4691e3a2ce80b59009ac17124f54f925f61dc5ea371903e62cdffa5d7b67ca96.json

@@ -1,50 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT * FROM session WHERE logged_out IS NULL OR logged_out > CURRENT_TIMESTAMP",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 1,
-        "name": "logged_in",
-        "type_info": "Timestamp"
-      },
-      {
-        "ordinal": 2,
-        "name": "logged_out",
-        "type_info": "Timestamp"
-      },
-      {
-        "ordinal": 3,
-        "name": "last_active",
-        "type_info": "Timestamp"
-      },
-      {
-        "ordinal": 4,
-        "name": "user_agent",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 5,
-        "name": "metadata",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      false,
-      true,
-      false,
-      true,
-      false
-    ]
-  },
-  "hash": "4691e3a2ce80b59009ac17124f54f925f61dc5ea371903e62cdffa5d7b67ca96"
-}

backend/.sqlx/query-4bcfbefb1eb3181343871a1cd7fc3afb81c2be5c681cfa8b4be0ce70610e9c3a.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE session SET logged_out = CURRENT_TIMESTAMP WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "4bcfbefb1eb3181343871a1cd7fc3afb81c2be5c681cfa8b4be0ce70610e9c3a"
-}

backend/.sqlx/query-629be61c3c341c131ddbbff0293a83dbc6afd07cae69d246987f62cf0cc35c2a.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT password FROM account",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "password",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "629be61c3c341c131ddbbff0293a83dbc6afd07cae69d246987f62cf0cc35c2a"
-}

backend/.sqlx/query-687688055e63d27123cdc89a5bbbd8361776290a9411d527eaf1fdb40bef399d.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT key FROM tor WHERE package = $1 AND interface = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "key",
-        "type_info": "Bytea"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "687688055e63d27123cdc89a5bbbd8361776290a9411d527eaf1fdb40bef399d"
-}

backend/.sqlx/query-6d35ccf780fb2bb62586dd1d3df9c1550a41ee580dad3f49d35cb843ebef10ca.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE session SET last_active = CURRENT_TIMESTAMP WHERE id = $1 AND logged_out IS NULL OR logged_out > CURRENT_TIMESTAMP",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "6d35ccf780fb2bb62586dd1d3df9c1550a41ee580dad3f49d35cb843ebef10ca"
-}

backend/.sqlx/query-770c1017734720453dc87b58c385b987c5af5807151ff71a59000014586752e0.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO UPDATE SET package = EXCLUDED.package RETURNING key",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "key",
-        "type_info": "Bytea"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Bytea"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "770c1017734720453dc87b58c385b987c5af5807151ff71a59000014586752e0"
-}

backend/.sqlx/query-7b64f032d507e8ffe37c41f4c7ad514a66c421a11ab04c26d89a7aa8f6b67210.json

@@ -1,65 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT id, package_id, created_at, code, level, title, message, data FROM notifications WHERE id < $1 ORDER BY id DESC LIMIT $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 1,
-        "name": "package_id",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "created_at",
-        "type_info": "Timestamp"
-      },
-      {
-        "ordinal": 3,
-        "name": "code",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 4,
-        "name": "level",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 5,
-        "name": "title",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 6,
-        "name": "message",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 7,
-        "name": "data",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false,
-      true,
-      false,
-      false,
-      false,
-      false,
-      false,
-      true
-    ]
-  },
-  "hash": "7b64f032d507e8ffe37c41f4c7ad514a66c421a11ab04c26d89a7aa8f6b67210"
-}

backend/.sqlx/query-7c7a3549c997eb75bf964ea65fbb98a73045adf618696cd838d79203ef5383fb.json

@@ -1,19 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            INSERT INTO account (\n                id,\n                server_id,\n                hostname,\n                password,\n                network_key,\n                root_ca_key_pem,\n                root_ca_cert_pem\n            ) VALUES (\n                0, $1, $2, $3, $4, $5, $6\n            ) ON CONFLICT (id) DO UPDATE SET\n                server_id = EXCLUDED.server_id,\n                hostname = EXCLUDED.hostname,\n                password = EXCLUDED.password,\n                network_key = EXCLUDED.network_key,\n                root_ca_key_pem = EXCLUDED.root_ca_key_pem,\n                root_ca_cert_pem = EXCLUDED.root_ca_cert_pem\n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text",
-        "Bytea",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "7c7a3549c997eb75bf964ea65fbb98a73045adf618696cd838d79203ef5383fb"
-}

backend/.sqlx/query-7e0649d839927e57fa03ee51a2c9f96a8bdb0fc97ee8a3c6df1069e1e2b98576.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM tor WHERE package = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "7e0649d839927e57fa03ee51a2c9f96a8bdb0fc97ee8a3c6df1069e1e2b98576"
-}

backend/.sqlx/query-8951b9126fbf60dbb5997241e11e3526b70bccf3e407327917294a993bc17ed5.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Bytea"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "8951b9126fbf60dbb5997241e11e3526b70bccf3e407327917294a993bc17ed5"
-}

backend/.sqlx/query-94d471bb374b4965c6cbedf8c17bbf6bea226d38efaf6559923c79a36d5ca08c.json

@@ -1,64 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT id, package_id, created_at, code, level, title, message, data FROM notifications ORDER BY id DESC LIMIT $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 1,
-        "name": "package_id",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "created_at",
-        "type_info": "Timestamp"
-      },
-      {
-        "ordinal": 3,
-        "name": "code",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 4,
-        "name": "level",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 5,
-        "name": "title",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 6,
-        "name": "message",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 7,
-        "name": "data",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false,
-      true,
-      false,
-      false,
-      false,
-      false,
-      false,
-      true
-    ]
-  },
-  "hash": "94d471bb374b4965c6cbedf8c17bbf6bea226d38efaf6559923c79a36d5ca08c"
-}

backend/.sqlx/query-95c4ab4c645f3302568c6ff13d85ab58252362694cf0f56999bf60194d20583a.json

@@ -1,44 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT id, hostname, path, username, password FROM cifs_shares",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 1,
-        "name": "hostname",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "path",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 3,
-        "name": "username",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 4,
-        "name": "password",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false,
-      true
-    ]
-  },
-  "hash": "95c4ab4c645f3302568c6ff13d85ab58252362694cf0f56999bf60194d20583a"
-}

backend/.sqlx/query-a60d6e66719325b08dc4ecfacaf337527233c84eee758ac9be967906e5841d27.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM cifs_shares WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "a60d6e66719325b08dc4ecfacaf337527233c84eee758ac9be967906e5841d27"
-}

backend/.sqlx/query-a6b0c8909a3a5d6d9156aebfb359424e6b5a1d1402e028219e21726f1ebd282e.json

@@ -1,32 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT fingerprint, openssh_pubkey, created_at FROM ssh_keys",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "fingerprint",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 1,
-        "name": "openssh_pubkey",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "created_at",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "a6b0c8909a3a5d6d9156aebfb359424e6b5a1d1402e028219e21726f1ebd282e"
-}

backend/.sqlx/query-b1147beaaabbed89f2ab8c1e13ec4393a9a8fde2833cf096af766a979d94dee6.json

@@ -1,18 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE cifs_shares SET hostname = $1, path = $2, username = $3, password = $4 WHERE id = $5",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text",
-        "Text",
-        "Int4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "b1147beaaabbed89f2ab8c1e13ec4393a9a8fde2833cf096af766a979d94dee6"
-}

backend/.sqlx/query-b203820ee1c553a4b246eac74b79bd10d5717b2a0ddecf22330b7d531aac7c5d.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM network_keys WHERE package = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "b203820ee1c553a4b246eac74b79bd10d5717b2a0ddecf22330b7d531aac7c5d"
-}

backend/.sqlx/query-d5117054072476377f3c4f040ea429d4c9b2cf534e76f35c80a2bf60e8599cca.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT openssh_pubkey FROM ssh_keys",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "openssh_pubkey",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "d5117054072476377f3c4f040ea429d4c9b2cf534e76f35c80a2bf60e8599cca"
-}

backend/.sqlx/query-da71f94b29798d1738d2b10b9a721ea72db8cfb362e7181c8226d9297507c62b.json

@@ -1,19 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO notifications (package_id, code, level, title, message, data) VALUES ($1, $2, $3, $4, $5, $6)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Int4",
-        "Text",
-        "Text",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "da71f94b29798d1738d2b10b9a721ea72db8cfb362e7181c8226d9297507c62b"
-}

backend/.sqlx/query-e185203cf84e43b801dfb23b4159e34aeaef1154dcd3d6811ab504915497ccf7.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM notifications WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "e185203cf84e43b801dfb23b4159e34aeaef1154dcd3d6811ab504915497ccf7"
-}

backend/.sqlx/query-e545696735f202f9d13cf22a561f3ff3f9aed7f90027a9ba97634bcb47d772f0.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT tor_key FROM account WHERE id = 0",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "tor_key",
-        "type_info": "Bytea"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      true
-    ]
-  },
-  "hash": "e545696735f202f9d13cf22a561f3ff3f9aed7f90027a9ba97634bcb47d772f0"
-}

backend/.sqlx/query-e5843c5b0e7819b29aa1abf2266799bd4f82e761837b526a0972c3d4439a264d.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO session (id, user_agent, metadata) VALUES ($1, $2, $3)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "e5843c5b0e7819b29aa1abf2266799bd4f82e761837b526a0972c3d4439a264d"
-}

backend/.sqlx/query-e95322a8e2ae3b93f1e974b24c0b81803f1e9ec9e8ebbf15cafddfc1c5a028ed.json

@@ -1,40 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                SELECT\n                    network_keys.package,\n                    network_keys.interface,\n                    network_keys.key,\n                    tor.key AS \"tor_key?\"\n                FROM\n                    network_keys\n                LEFT JOIN\n                    tor\n                ON\n                    network_keys.package = tor.package\n                AND\n                    network_keys.interface = tor.interface\n                WHERE\n                    network_keys.package = $1\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "package",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 1,
-        "name": "interface",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "key",
-        "type_info": "Bytea"
-      },
-      {
-        "ordinal": 3,
-        "name": "tor_key?",
-        "type_info": "Bytea"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "e95322a8e2ae3b93f1e974b24c0b81803f1e9ec9e8ebbf15cafddfc1c5a028ed"
-}

backend/.sqlx/query-eb750adaa305bdbf3c5b70aaf59139c7b7569602adb58f2d6b3a94da4f167b0a.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM notifications WHERE id < $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "eb750adaa305bdbf3c5b70aaf59139c7b7569602adb58f2d6b3a94da4f167b0a"
-}

backend/.sqlx/query-ecc765d8205c0876956f95f76944ac6a5f34dd820c4073b7728c7067aab9fded.json

@@ -1,25 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO cifs_shares (hostname, path, username, password) VALUES ($1, $2, $3, $4) RETURNING id",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "ecc765d8205c0876956f95f76944ac6a5f34dd820c4073b7728c7067aab9fded"
-}

backend/.sqlx/query-f6d1c5ef0f9d9577bea8382318967b9deb46da75788c7fe6082b43821c22d556.json

@@ -1,16 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO ssh_keys (fingerprint, openssh_pubkey, created_at) VALUES ($1, $2, $3)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "f6d1c5ef0f9d9577bea8382318967b9deb46da75788c7fe6082b43821c22d556"
-}

backend/.sqlx/query-f7d2dae84613bcef330f7403352cc96547f3f6dbec11bf2eadfaf53ad8ab51b5.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT network_key FROM account WHERE id = 0",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "network_key",
-        "type_info": "Bytea"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "f7d2dae84613bcef330f7403352cc96547f3f6dbec11bf2eadfaf53ad8ab51b5"
-}

backend/.sqlx/query-fe6e4f09f3028e5b6b6259e86cbad285680ce157aae9d7837ac020c8b2945e7f.json

@@ -1,62 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT * FROM account WHERE id = 0",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      },
-      {
-        "ordinal": 1,
-        "name": "password",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "tor_key",
-        "type_info": "Bytea"
-      },
-      {
-        "ordinal": 3,
-        "name": "server_id",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 4,
-        "name": "hostname",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 5,
-        "name": "network_key",
-        "type_info": "Bytea"
-      },
-      {
-        "ordinal": 6,
-        "name": "root_ca_key_pem",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 7,
-        "name": "root_ca_cert_pem",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      false,
-      true,
-      true,
-      true,
-      false,
-      false,
-      false
-    ]
-  },
-  "hash": "fe6e4f09f3028e5b6b6259e86cbad285680ce157aae9d7837ac020c8b2945e7f"
-}

backend/README.md

@@ -1,42 +0,0 @@
-# StartOS Backend
-
-- Requirements:
-  - [Install Rust](https://rustup.rs)
-  - Recommended: [rust-analyzer](https://rust-analyzer.github.io/)
-  - [Docker](https://docs.docker.com/get-docker/)
-  - [Rust ARM64 Build Container](https://github.com/Start9Labs/rust-arm-builder)
-  - Mac `brew install gnu-tar`
-- Scripts (run within the `./backend` directory)
-  - `build-prod.sh` - compiles a release build of the artifacts for running on
-    ARM64
-- A Linux computer or VM
-
-## Structure
-
-The StartOS backend is packed into a single binary `startbox` that is symlinked under
-several different names for different behaviour:
-
-- startd: This is the main workhorse of StartOS - any new functionality you
-  want will likely go here
-- start-cli: This is a CLI tool that will allow you to issue commands to
-  startd and control it similarly to the UI
-- start-sdk: This is a CLI tool that aids in building and packaging services
-  you wish to deploy to StartOS
-
-Finally there is a library `startos` that supports all of these tools.
-
-See [here](/backend/Cargo.toml) for details.
-
-## Building
-
-You can build the entire operating system image using `make` from the root of
-the StartOS project. This will subsequently invoke the build scripts above to
-actually create the requisite binaries and put them onto the final operating
-system image.
-
-## Questions
-
-If you have questions about how various pieces of the backend system work. Open
-an issue and tag the following people
-
-- dr-bonez

backend/build-portable.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -e
-shopt -s expand_aliases
-
-if [ "$0" != "./build-portable.sh" ]; then
-	>&2 echo "Must be run from backend directory"
-	exit 1
-fi
-
-USE_TTY=
-if tty -s; then
-	USE_TTY="-it"
-fi
-
-alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:x86_64-musl'
-
-cd ..
-rust-musl-builder sh -c "(cd backend && cargo +beta build --release --target=x86_64-unknown-linux-musl --no-default-features --locked)"
-cd backend
-
-sudo chown -R $USER target
-sudo chown -R $USER ~/.cargo

backend/build-prod.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-
-set -e
-shopt -s expand_aliases
-
-if [ -z "$ARCH" ]; then
-	ARCH=$(uname -m)
-fi
-
-if [ "$0" != "./build-prod.sh" ]; then
-	>&2 echo "Must be run from backend directory"
-	exit 1
-fi
-
-USE_TTY=
-if tty -s; then
-	USE_TTY="-it"
-fi
-
-cd ..
-FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
-RUSTFLAGS=""
-
-alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
-alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
-
-set +e
-fail=
-echo "FEATURES=\"$FEATURES\""
-echo "RUSTFLAGS=\"$RUSTFLAGS\""
-rust-gnu-builder sh -c "(cd backend && cargo build --release --features avahi-alias,$FEATURES --locked --target=$ARCH-unknown-linux-gnu)"
-if test $? -ne 0; then 
-	fail=true
-fi
-for ARCH in x86_64 aarch64
-do
-	rust-musl-builder sh -c "(cd libs && cargo build --release --locked --bin embassy_container_init)"
-	if test $? -ne 0; then 
-		fail=true
-	fi
-done
-set -e
-cd backend
-
-sudo chown -R $USER target
-sudo chown -R $USER ~/.cargo
-sudo chown -R $USER ../libs/target
-
-if [ -n "$fail" ]; then
-	exit 1
-fi

backend/install-sdk.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e
-shopt -s expand_aliases
-
-if [ "$0" != "./install-sdk.sh" ]; then
-	>&2 echo "Must be run from backend directory"
-	exit 1
-fi
-
-frontend="../frontend/dist/static"
-[ -d "$frontend" ] || mkdir -p "$frontend"
-
-if [ -z "$PLATFORM" ]; then
-  export PLATFORM=$(uname -m)
-fi
-
-cargo install --path=. --no-default-features --features=js_engine,sdk,cli --locked
-startbox_loc=$(which startbox)
-ln -sf $startbox_loc $(dirname $startbox_loc)/start-cli
-ln -sf $startbox_loc $(dirname $startbox_loc)/start-sdk

backend/src/account.rs

@@ -1,132 +0,0 @@
-use std::time::SystemTime;
-
-use ed25519_dalek::SecretKey;
-use openssl::pkey::{PKey, Private};
-use openssl::x509::X509;
-use sqlx::PgExecutor;
-
-use crate::hostname::{generate_hostname, generate_id, Hostname};
-use crate::net::keys::Key;
-use crate::net::ssl::{generate_key, make_root_cert};
-use crate::prelude::*;
-use crate::util::crypto::ed25519_expand_key;
-
-fn hash_password(password: &str) -> Result<String, Error> {
-    argon2::hash_encoded(
-        password.as_bytes(),
-        &rand::random::<[u8; 16]>()[..],
-        &argon2::Config::rfc9106_low_mem(),
-    )
-    .with_kind(crate::ErrorKind::PasswordHashGeneration)
-}
-
-#[derive(Debug, Clone)]
-pub struct AccountInfo {
-    pub server_id: String,
-    pub hostname: Hostname,
-    pub password: String,
-    pub key: Key,
-    pub root_ca_key: PKey<Private>,
-    pub root_ca_cert: X509,
-}
-impl AccountInfo {
-    pub fn new(password: &str, start_time: SystemTime) -> Result<Self, Error> {
-        let server_id = generate_id();
-        let hostname = generate_hostname();
-        let root_ca_key = generate_key()?;
-        let root_ca_cert = make_root_cert(&root_ca_key, &hostname, start_time)?;
-        Ok(Self {
-            server_id,
-            hostname,
-            password: hash_password(password)?,
-            key: Key::new(None),
-            root_ca_key,
-            root_ca_cert,
-        })
-    }
-
-    pub async fn load(secrets: impl PgExecutor<'_>) -> Result<Self, Error> {
-        let r = sqlx::query!("SELECT * FROM account WHERE id = 0")
-            .fetch_one(secrets)
-            .await?;
-
-        let server_id = r.server_id.unwrap_or_else(generate_id);
-        let hostname = r.hostname.map(Hostname).unwrap_or_else(generate_hostname);
-        let password = r.password;
-        let network_key = SecretKey::try_from(r.network_key).map_err(|e| {
-            Error::new(
-                eyre!("expected vec of len 32, got len {}", e.len()),
-                ErrorKind::ParseDbField,
-            )
-        })?;
-        let tor_key = if let Some(k) = &r.tor_key {
-            <[u8; 64]>::try_from(&k[..]).map_err(|_| {
-                Error::new(
-                    eyre!("expected vec of len 64, got len {}", k.len()),
-                    ErrorKind::ParseDbField,
-                )
-            })?
-        } else {
-            ed25519_expand_key(&network_key)
-        };
-        let key = Key::from_pair(None, network_key, tor_key);
-        let root_ca_key = PKey::private_key_from_pem(r.root_ca_key_pem.as_bytes())?;
-        let root_ca_cert = X509::from_pem(r.root_ca_cert_pem.as_bytes())?;
-
-        Ok(Self {
-            server_id,
-            hostname,
-            password,
-            key,
-            root_ca_key,
-            root_ca_cert,
-        })
-    }
-
-    pub async fn save(&self, secrets: impl PgExecutor<'_>) -> Result<(), Error> {
-        let server_id = self.server_id.as_str();
-        let hostname = self.hostname.0.as_str();
-        let password = self.password.as_str();
-        let network_key = self.key.as_bytes();
-        let network_key = network_key.as_slice();
-        let root_ca_key = String::from_utf8(self.root_ca_key.private_key_to_pem_pkcs8()?)?;
-        let root_ca_cert = String::from_utf8(self.root_ca_cert.to_pem()?)?;
-
-        sqlx::query!(
-            r#"
-            INSERT INTO account (
-                id,
-                server_id,
-                hostname,
-                password,
-                network_key,
-                root_ca_key_pem,
-                root_ca_cert_pem
-            ) VALUES (
-                0, $1, $2, $3, $4, $5, $6
-            ) ON CONFLICT (id) DO UPDATE SET
-                server_id = EXCLUDED.server_id,
-                hostname = EXCLUDED.hostname,
-                password = EXCLUDED.password,
-                network_key = EXCLUDED.network_key,
-                root_ca_key_pem = EXCLUDED.root_ca_key_pem,
-                root_ca_cert_pem = EXCLUDED.root_ca_cert_pem
-            "#,
-            server_id,
-            hostname,
-            password,
-            network_key,
-            root_ca_key,
-            root_ca_cert,
-        )
-        .execute(secrets)
-        .await?;
-
-        Ok(())
-    }
-
-    pub fn set_password(&mut self, password: &str) -> Result<(), Error> {
-        self.password = hash_password(password)?;
-        Ok(())
-    }
-}

backend/src/action.rs

@@ -1,163 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet};
-
-use clap::ArgMatches;
-use color_eyre::eyre::eyre;
-use indexmap::IndexSet;
-pub use models::ActionId;
-use models::ImageId;
-use rpc_toolkit::command;
-use serde::{Deserialize, Serialize};
-use tracing::instrument;
-
-use crate::config::{Config, ConfigSpec};
-use crate::context::RpcContext;
-use crate::prelude::*;
-use crate::procedure::docker::DockerContainers;
-use crate::procedure::{PackageProcedure, ProcedureName};
-use crate::s9pk::manifest::PackageId;
-use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
-use crate::util::Version;
-use crate::volume::Volumes;
-use crate::{Error, ResultExt};
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub struct Actions(pub BTreeMap<ActionId, Action>);
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "version")]
-pub enum ActionResult {
-    #[serde(rename = "0")]
-    V0(ActionResultV0),
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct ActionResultV0 {
-    pub message: String,
-    pub value: Option<String>,
-    pub copyable: bool,
-    pub qr: bool,
-}
-
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub enum DockerStatus {
-    Running,
-    Stopped,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct Action {
-    pub name: String,
-    pub description: String,
-    #[serde(default)]
-    pub warning: Option<String>,
-    pub implementation: PackageProcedure,
-    pub allowed_statuses: IndexSet<DockerStatus>,
-    #[serde(default)]
-    pub input_spec: ConfigSpec,
-}
-impl Action {
-    #[instrument(skip_all)]
-    pub fn validate(
-        &self,
-        _container: &Option<DockerContainers>,
-        eos_version: &Version,
-        volumes: &Volumes,
-        image_ids: &BTreeSet<ImageId>,
-    ) -> Result<(), Error> {
-        self.implementation
-            .validate(eos_version, volumes, image_ids, true)
-            .with_ctx(|_| {
-                (
-                    crate::ErrorKind::ValidateS9pk,
-                    format!("Action {}", self.name),
-                )
-            })
-    }
-
-    #[instrument(skip_all)]
-    pub async fn execute(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        action_id: &ActionId,
-        volumes: &Volumes,
-        input: Option<Config>,
-    ) -> Result<ActionResult, Error> {
-        if let Some(ref input) = input {
-            self.input_spec
-                .matches(&input)
-                .with_kind(crate::ErrorKind::ConfigSpecViolation)?;
-        }
-        self.implementation
-            .execute(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::Action(action_id.clone()),
-                volumes,
-                input,
-                None,
-            )
-            .await?
-            .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::Action))
-    }
-}
-
-fn display_action_result(action_result: ActionResult, matches: &ArgMatches) {
-    if matches.is_present("format") {
-        return display_serializable(action_result, matches);
-    }
-    match action_result {
-        ActionResult::V0(ar) => {
-            println!(
-                "{}: {}",
-                ar.message,
-                serde_json::to_string(&ar.value).unwrap()
-            );
-        }
-    }
-}
-
-#[command(about = "Executes an action", display(display_action_result))]
-#[instrument(skip_all)]
-pub async fn action(
-    #[context] ctx: RpcContext,
-    #[arg(rename = "id")] pkg_id: PackageId,
-    #[arg(rename = "action-id")] action_id: ActionId,
-    #[arg(stdin, parse(parse_stdin_deserializable))] input: Option<Config>,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-) -> Result<ActionResult, Error> {
-    let manifest = ctx
-        .db
-        .peek()
-        .await
-        .as_package_data()
-        .as_idx(&pkg_id)
-        .or_not_found(&pkg_id)?
-        .as_installed()
-        .or_not_found(&pkg_id)?
-        .as_manifest()
-        .de()?;
-
-    if let Some(action) = manifest.actions.0.get(&action_id) {
-        action
-            .execute(
-                &ctx,
-                &manifest.id,
-                &manifest.version,
-                &action_id,
-                &manifest.volumes,
-                input,
-            )
-            .await
-    } else {
-        Err(Error::new(
-            eyre!("Action not found in manifest"),
-            crate::ErrorKind::NotFound,
-        ))
-    }
-}

backend/src/auth.rs

@@ -1,391 +0,0 @@
-use std::collections::BTreeMap;
-use std::marker::PhantomData;
-
-use chrono::{DateTime, Utc};
-use clap::ArgMatches;
-use color_eyre::eyre::eyre;
-use josekit::jwk::Jwk;
-use rpc_toolkit::command;
-use rpc_toolkit::command_helpers::prelude::{RequestParts, ResponseParts};
-use rpc_toolkit::yajrc::RpcError;
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use sqlx::{Executor, Postgres};
-use tracing::instrument;
-
-use crate::context::{CliContext, RpcContext};
-use crate::middleware::auth::{AsLogoutSessionId, HasLoggedOutSessions, HashSessionToken};
-use crate::middleware::encrypt::EncryptedWire;
-use crate::prelude::*;
-use crate::util::display_none;
-use crate::util::serde::{display_serializable, IoFormat};
-use crate::{ensure_code, Error, ResultExt};
-#[derive(Clone, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum PasswordType {
-    EncryptedWire(EncryptedWire),
-    String(String),
-}
-impl PasswordType {
-    pub fn decrypt(self, current_secret: impl AsRef<Jwk>) -> Result<String, Error> {
-        match self {
-            PasswordType::String(x) => Ok(x),
-            PasswordType::EncryptedWire(x) => x.decrypt(current_secret).ok_or_else(|| {
-                Error::new(
-                    color_eyre::eyre::eyre!("Couldn't decode password"),
-                    crate::ErrorKind::Unknown,
-                )
-            }),
-        }
-    }
-}
-impl Default for PasswordType {
-    fn default() -> Self {
-        PasswordType::String(String::default())
-    }
-}
-impl std::fmt::Debug for PasswordType {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "<REDACTED_PASSWORD>")?;
-        Ok(())
-    }
-}
-
-impl std::str::FromStr for PasswordType {
-    type Err = String;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(match serde_json::from_str(s) {
-            Ok(a) => a,
-            Err(_) => PasswordType::String(s.to_string()),
-        })
-    }
-}
-
-#[command(subcommands(login, logout, session, reset_password, get_pubkey))]
-pub fn auth() -> Result<(), Error> {
-    Ok(())
-}
-
-pub fn cli_metadata() -> Value {
-    serde_json::json!({
-        "platforms": ["cli"],
-    })
-}
-
-pub fn parse_metadata(_: &str, _: &ArgMatches) -> Result<Value, Error> {
-    Ok(cli_metadata())
-}
-
-#[test]
-fn gen_pwd() {
-    println!(
-        "{:?}",
-        argon2::hash_encoded(
-            b"testing1234",
-            &rand::random::<[u8; 16]>()[..],
-            &argon2::Config::rfc9106_low_mem()
-        )
-        .unwrap()
-    )
-}
-
-#[instrument(skip_all)]
-async fn cli_login(
-    ctx: CliContext,
-    password: Option<PasswordType>,
-    metadata: Value,
-) -> Result<(), RpcError> {
-    let password = if let Some(password) = password {
-        password.decrypt(&ctx)?
-    } else {
-        rpassword::prompt_password("Password: ")?
-    };
-
-    rpc_toolkit::command_helpers::call_remote(
-        ctx,
-        "auth.login",
-        serde_json::json!({ "password": password, "metadata": metadata }),
-        PhantomData::<()>,
-    )
-    .await?
-    .result?;
-
-    Ok(())
-}
-
-pub fn check_password(hash: &str, password: &str) -> Result<(), Error> {
-    ensure_code!(
-        argon2::verify_encoded(&hash, password.as_bytes()).map_err(|_| {
-            Error::new(
-                eyre!("Password Incorrect"),
-                crate::ErrorKind::IncorrectPassword,
-            )
-        })?,
-        crate::ErrorKind::IncorrectPassword,
-        "Password Incorrect"
-    );
-    Ok(())
-}
-
-pub async fn check_password_against_db<Ex>(secrets: &mut Ex, password: &str) -> Result<(), Error>
-where
-    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-{
-    let pw_hash = sqlx::query!("SELECT password FROM account")
-        .fetch_one(secrets)
-        .await?
-        .password;
-    check_password(&pw_hash, password)?;
-    Ok(())
-}
-
-#[command(
-    custom_cli(cli_login(async, context(CliContext))),
-    display(display_none),
-    metadata(authenticated = false)
-)]
-#[instrument(skip_all)]
-pub async fn login(
-    #[context] ctx: RpcContext,
-    #[request] req: &RequestParts,
-    #[response] res: &mut ResponseParts,
-    #[arg] password: Option<PasswordType>,
-    #[arg(
-        parse(parse_metadata),
-        default = "cli_metadata",
-        help = "RPC Only: This value cannot be overidden from the cli"
-    )]
-    metadata: Value,
-) -> Result<(), Error> {
-    let password = password.unwrap_or_default().decrypt(&ctx)?;
-    let mut handle = ctx.secret_store.acquire().await?;
-    check_password_against_db(handle.as_mut(), &password).await?;
-
-    let hash_token = HashSessionToken::new();
-    let user_agent = req.headers.get("user-agent").and_then(|h| h.to_str().ok());
-    let metadata = serde_json::to_string(&metadata).with_kind(crate::ErrorKind::Database)?;
-    let hash_token_hashed = hash_token.hashed();
-    sqlx::query!(
-        "INSERT INTO session (id, user_agent, metadata) VALUES ($1, $2, $3)",
-        hash_token_hashed,
-        user_agent,
-        metadata,
-    )
-    .execute(handle.as_mut())
-    .await?;
-    res.headers.insert(
-        "set-cookie",
-        hash_token.header_value()?, // Should be impossible, but don't want to panic
-    );
-
-    Ok(())
-}
-
-#[command(display(display_none), metadata(authenticated = false))]
-#[instrument(skip_all)]
-pub async fn logout(
-    #[context] ctx: RpcContext,
-    #[request] req: &RequestParts,
-) -> Result<Option<HasLoggedOutSessions>, Error> {
-    let auth = match HashSessionToken::from_request_parts(req) {
-        Err(_) => return Ok(None),
-        Ok(a) => a,
-    };
-    Ok(Some(HasLoggedOutSessions::new(vec![auth], &ctx).await?))
-}
-
-#[derive(Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct Session {
-    logged_in: DateTime<Utc>,
-    last_active: DateTime<Utc>,
-    user_agent: Option<String>,
-    metadata: Value,
-}
-
-#[derive(Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct SessionList {
-    current: String,
-    sessions: BTreeMap<String, Session>,
-}
-
-#[command(subcommands(list, kill))]
-pub async fn session() -> Result<(), Error> {
-    Ok(())
-}
-
-fn display_sessions(arg: SessionList, matches: &ArgMatches) {
-    use prettytable::*;
-
-    if matches.is_present("format") {
-        return display_serializable(arg, matches);
-    }
-
-    let mut table = Table::new();
-    table.add_row(row![bc =>
-        "ID",
-        "LOGGED IN",
-        "LAST ACTIVE",
-        "USER AGENT",
-        "METADATA",
-    ]);
-    for (id, session) in arg.sessions {
-        let mut row = row![
-            &id,
-            &format!("{}", session.logged_in),
-            &format!("{}", session.last_active),
-            session.user_agent.as_deref().unwrap_or("N/A"),
-            &format!("{}", session.metadata),
-        ];
-        if id == arg.current {
-            row.iter_mut()
-                .map(|c| c.style(Attr::ForegroundColor(color::GREEN)))
-                .collect::<()>()
-        }
-        table.add_row(row);
-    }
-    table.print_tty(false).unwrap();
-}
-
-#[command(display(display_sessions))]
-#[instrument(skip_all)]
-pub async fn list(
-    #[context] ctx: RpcContext,
-    #[request] req: &RequestParts,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-) -> Result<SessionList, Error> {
-    Ok(SessionList {
-        current: HashSessionToken::from_request_parts(req)?.as_hash(),
-        sessions: sqlx::query!(
-            "SELECT * FROM session WHERE logged_out IS NULL OR logged_out > CURRENT_TIMESTAMP"
-        )
-        .fetch_all(ctx.secret_store.acquire().await?.as_mut())
-        .await?
-        .into_iter()
-        .map(|row| {
-            Ok((
-                row.id,
-                Session {
-                    logged_in: DateTime::from_utc(row.logged_in, Utc),
-                    last_active: DateTime::from_utc(row.last_active, Utc),
-                    user_agent: row.user_agent,
-                    metadata: serde_json::from_str(&row.metadata)
-                        .with_kind(crate::ErrorKind::Database)?,
-                },
-            ))
-        })
-        .collect::<Result<_, Error>>()?,
-    })
-}
-
-fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<String>, RpcError> {
-    Ok(arg.split(",").map(|s| s.trim().to_owned()).collect())
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct KillSessionId(String);
-
-impl AsLogoutSessionId for KillSessionId {
-    fn as_logout_session_id(self) -> String {
-        self.0
-    }
-}
-
-#[command(display(display_none))]
-#[instrument(skip_all)]
-pub async fn kill(
-    #[context] ctx: RpcContext,
-    #[arg(parse(parse_comma_separated))] ids: Vec<String>,
-) -> Result<(), Error> {
-    HasLoggedOutSessions::new(ids.into_iter().map(KillSessionId), &ctx).await?;
-    Ok(())
-}
-
-#[instrument(skip_all)]
-async fn cli_reset_password(
-    ctx: CliContext,
-    old_password: Option<PasswordType>,
-    new_password: Option<PasswordType>,
-) -> Result<(), RpcError> {
-    let old_password = if let Some(old_password) = old_password {
-        old_password.decrypt(&ctx)?
-    } else {
-        rpassword::prompt_password("Current Password: ")?
-    };
-
-    let new_password = if let Some(new_password) = new_password {
-        new_password.decrypt(&ctx)?
-    } else {
-        let new_password = rpassword::prompt_password("New Password: ")?;
-        if new_password != rpassword::prompt_password("Confirm: ")? {
-            return Err(Error::new(
-                eyre!("Passwords do not match"),
-                crate::ErrorKind::IncorrectPassword,
-            )
-            .into());
-        }
-        new_password
-    };
-
-    rpc_toolkit::command_helpers::call_remote(
-        ctx,
-        "auth.reset-password",
-        serde_json::json!({ "old-password": old_password, "new-password": new_password }),
-        PhantomData::<()>,
-    )
-    .await?
-    .result?;
-
-    Ok(())
-}
-
-#[command(
-    rename = "reset-password",
-    custom_cli(cli_reset_password(async, context(CliContext))),
-    display(display_none)
-)]
-#[instrument(skip_all)]
-pub async fn reset_password(
-    #[context] ctx: RpcContext,
-    #[arg(rename = "old-password")] old_password: Option<PasswordType>,
-    #[arg(rename = "new-password")] new_password: Option<PasswordType>,
-) -> Result<(), Error> {
-    let old_password = old_password.unwrap_or_default().decrypt(&ctx)?;
-    let new_password = new_password.unwrap_or_default().decrypt(&ctx)?;
-
-    let mut account = ctx.account.write().await;
-    if !argon2::verify_encoded(&account.password, old_password.as_bytes())
-        .with_kind(crate::ErrorKind::IncorrectPassword)?
-    {
-        return Err(Error::new(
-            eyre!("Incorrect Password"),
-            crate::ErrorKind::IncorrectPassword,
-        ));
-    }
-    account.set_password(&new_password)?;
-    account.save(&ctx.secret_store).await?;
-    let account_password = &account.password;
-    ctx.db
-        .mutate(|d| {
-            d.as_server_info_mut()
-                .as_password_hash_mut()
-                .ser(account_password)
-        })
-        .await
-}
-
-#[command(
-    rename = "get-pubkey",
-    display(display_none),
-    metadata(authenticated = false)
-)]
-#[instrument(skip_all)]
-pub async fn get_pubkey(#[context] ctx: RpcContext) -> Result<Jwk, RpcError> {
-    let secret = ctx.as_ref().clone();
-    let pub_key = secret.to_public_key()?;
-    Ok(pub_key)
-}

backend/src/backup/backup_bulk.rs

@@ -1,322 +0,0 @@
-use std::collections::BTreeMap;
-use std::panic::UnwindSafe;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use chrono::Utc;
-use clap::ArgMatches;
-use color_eyre::eyre::eyre;
-use helpers::AtomicFile;
-use imbl::OrdSet;
-use models::Version;
-use rpc_toolkit::command;
-use tokio::io::AsyncWriteExt;
-use tokio::sync::Mutex;
-use tracing::instrument;
-
-use super::target::BackupTargetId;
-use super::PackageBackupReport;
-use crate::auth::check_password_against_db;
-use crate::backup::os::OsBackup;
-use crate::backup::{BackupReport, ServerBackupReport};
-use crate::context::RpcContext;
-use crate::db::model::BackupProgress;
-use crate::db::package::get_packages;
-use crate::disk::mount::backup::BackupMountGuard;
-use crate::disk::mount::filesystem::ReadWrite;
-use crate::disk::mount::guard::TmpMountGuard;
-use crate::manager::BackupReturn;
-use crate::notifications::NotificationLevel;
-use crate::prelude::*;
-use crate::s9pk::manifest::PackageId;
-use crate::util::display_none;
-use crate::util::io::dir_copy;
-use crate::util::serde::IoFormat;
-use crate::version::VersionT;
-
-fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<OrdSet<PackageId>, Error> {
-    arg.split(',')
-        .map(|s| s.trim().parse::<PackageId>().map_err(Error::from))
-        .collect()
-}
-
-#[command(rename = "create", display(display_none))]
-#[instrument(skip(ctx, old_password, password))]
-pub async fn backup_all(
-    #[context] ctx: RpcContext,
-    #[arg(rename = "target-id")] target_id: BackupTargetId,
-    #[arg(rename = "old-password", long = "old-password")] old_password: Option<
-        crate::auth::PasswordType,
-    >,
-    #[arg(
-        rename = "package-ids",
-        long = "package-ids",
-        parse(parse_comma_separated)
-    )]
-    package_ids: Option<OrdSet<PackageId>>,
-    #[arg] password: crate::auth::PasswordType,
-) -> Result<(), Error> {
-    let db = ctx.db.peek().await;
-    let old_password_decrypted = old_password
-        .as_ref()
-        .unwrap_or(&password)
-        .clone()
-        .decrypt(&ctx)?;
-    let password = password.decrypt(&ctx)?;
-    check_password_against_db(ctx.secret_store.acquire().await?.as_mut(), &password).await?;
-    let fs = target_id
-        .load(ctx.secret_store.acquire().await?.as_mut())
-        .await?;
-    let mut backup_guard = BackupMountGuard::mount(
-        TmpMountGuard::mount(&fs, ReadWrite).await?,
-        &old_password_decrypted,
-    )
-    .await?;
-    let package_ids = if let Some(ids) = package_ids {
-        ids.into_iter()
-            .flat_map(|package_id| {
-                let version = db
-                    .as_package_data()
-                    .as_idx(&package_id)?
-                    .as_manifest()
-                    .as_version()
-                    .de()
-                    .ok()?;
-                Some((package_id, version))
-            })
-            .collect()
-    } else {
-        get_packages(db.clone())?.into_iter().collect()
-    };
-    if old_password.is_some() {
-        backup_guard.change_password(&password)?;
-    }
-    assure_backing_up(&ctx.db, &package_ids).await?;
-    tokio::task::spawn(async move {
-        let backup_res = perform_backup(&ctx, backup_guard, &package_ids).await;
-        match backup_res {
-            Ok(report) if report.iter().all(|(_, rep)| rep.error.is_none()) => ctx
-                .notification_manager
-                .notify(
-                    ctx.db.clone(),
-                    None,
-                    NotificationLevel::Success,
-                    "Backup Complete".to_owned(),
-                    "Your backup has completed".to_owned(),
-                    BackupReport {
-                        server: ServerBackupReport {
-                            attempted: true,
-                            error: None,
-                        },
-                        packages: report
-                            .into_iter()
-                            .map(|((package_id, _), value)| (package_id, value))
-                            .collect(),
-                    },
-                    None,
-                )
-                .await
-                .expect("failed to send notification"),
-            Ok(report) => ctx
-                .notification_manager
-                .notify(
-                    ctx.db.clone(),
-                    None,
-                    NotificationLevel::Warning,
-                    "Backup Complete".to_owned(),
-                    "Your backup has completed, but some package(s) failed to backup".to_owned(),
-                    BackupReport {
-                        server: ServerBackupReport {
-                            attempted: true,
-                            error: None,
-                        },
-                        packages: report
-                            .into_iter()
-                            .map(|((package_id, _), value)| (package_id, value))
-                            .collect(),
-                    },
-                    None,
-                )
-                .await
-                .expect("failed to send notification"),
-            Err(e) => {
-                tracing::error!("Backup Failed: {}", e);
-                tracing::debug!("{:?}", e);
-                ctx.notification_manager
-                    .notify(
-                        ctx.db.clone(),
-                        None,
-                        NotificationLevel::Error,
-                        "Backup Failed".to_owned(),
-                        "Your backup failed to complete.".to_owned(),
-                        BackupReport {
-                            server: ServerBackupReport {
-                                attempted: true,
-                                error: Some(e.to_string()),
-                            },
-                            packages: BTreeMap::new(),
-                        },
-                        None,
-                    )
-                    .await
-                    .expect("failed to send notification");
-            }
-        }
-        ctx.db
-            .mutate(|v| {
-                v.as_server_info_mut()
-                    .as_status_info_mut()
-                    .as_backup_progress_mut()
-                    .ser(&None)
-            })
-            .await?;
-        Ok::<(), Error>(())
-    });
-    Ok(())
-}
-
-#[instrument(skip(db, packages))]
-async fn assure_backing_up(
-    db: &PatchDb,
-    packages: impl IntoIterator<Item = &(PackageId, Version)> + UnwindSafe + Send,
-) -> Result<(), Error> {
-    db.mutate(|v| {
-        let backing_up = v
-            .as_server_info_mut()
-            .as_status_info_mut()
-            .as_backup_progress_mut();
-        if backing_up
-            .clone()
-            .de()?
-            .iter()
-            .flat_map(|x| x.values())
-            .fold(false, |acc, x| {
-                if !x.complete {
-                    return true;
-                }
-                acc
-            })
-        {
-            return Err(Error::new(
-                eyre!("Server is already backing up!"),
-                ErrorKind::InvalidRequest,
-            ));
-        }
-        backing_up.ser(&Some(
-            packages
-                .into_iter()
-                .map(|(x, _)| (x.clone(), BackupProgress { complete: false }))
-                .collect(),
-        ))?;
-        Ok(())
-    })
-    .await
-}
-
-#[instrument(skip(ctx, backup_guard))]
-async fn perform_backup(
-    ctx: &RpcContext,
-    backup_guard: BackupMountGuard<TmpMountGuard>,
-    package_ids: &OrdSet<(PackageId, Version)>,
-) -> Result<BTreeMap<(PackageId, Version), PackageBackupReport>, Error> {
-    let mut backup_report = BTreeMap::new();
-    let backup_guard = Arc::new(Mutex::new(backup_guard));
-
-    for package_id in package_ids {
-        let (response, _report) = match ctx
-            .managers
-            .get(package_id)
-            .await
-            .ok_or_else(|| Error::new(eyre!("Manager not found"), ErrorKind::InvalidRequest))?
-            .backup(backup_guard.clone())
-            .await
-        {
-            BackupReturn::Ran { report, res } => (res, report),
-            BackupReturn::AlreadyRunning(report) => {
-                backup_report.insert(package_id.clone(), report);
-                continue;
-            }
-            BackupReturn::Error(error) => {
-                tracing::warn!("Backup thread error");
-                tracing::debug!("{error:?}");
-                backup_report.insert(
-                    package_id.clone(),
-                    PackageBackupReport {
-                        error: Some("Backup thread error".to_owned()),
-                    },
-                );
-                continue;
-            }
-        };
-        backup_report.insert(
-            package_id.clone(),
-            PackageBackupReport {
-                error: response.as_ref().err().map(|e| e.to_string()),
-            },
-        );
-
-        if let Ok(pkg_meta) = response {
-            backup_guard
-                .lock()
-                .await
-                .metadata
-                .package_backups
-                .insert(package_id.0.clone(), pkg_meta);
-        }
-    }
-
-    let ui = ctx.db.peek().await.into_ui().de()?;
-
-    let mut os_backup_file = AtomicFile::new(
-        backup_guard.lock().await.as_ref().join("os-backup.cbor"),
-        None::<PathBuf>,
-    )
-    .await
-    .with_kind(ErrorKind::Filesystem)?;
-    os_backup_file
-        .write_all(&IoFormat::Cbor.to_vec(&OsBackup {
-            account: ctx.account.read().await.clone(),
-            ui,
-        })?)
-        .await?;
-    os_backup_file
-        .save()
-        .await
-        .with_kind(ErrorKind::Filesystem)?;
-
-    let luks_folder_old = backup_guard.lock().await.as_ref().join("luks.old");
-    if tokio::fs::metadata(&luks_folder_old).await.is_ok() {
-        tokio::fs::remove_dir_all(&luks_folder_old).await?;
-    }
-    let luks_folder_bak = backup_guard.lock().await.as_ref().join("luks");
-    if tokio::fs::metadata(&luks_folder_bak).await.is_ok() {
-        tokio::fs::rename(&luks_folder_bak, &luks_folder_old).await?;
-    }
-    let luks_folder = Path::new("/media/embassy/config/luks");
-    if tokio::fs::metadata(&luks_folder).await.is_ok() {
-        dir_copy(&luks_folder, &luks_folder_bak, None).await?;
-    }
-
-    let timestamp = Some(Utc::now());
-    let mut backup_guard = Arc::try_unwrap(backup_guard)
-        .map_err(|_err| {
-            Error::new(
-                eyre!("Backup guard could not ensure that the others where dropped"),
-                ErrorKind::Unknown,
-            )
-        })?
-        .into_inner();
-
-    backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();
-    backup_guard.unencrypted_metadata.full = true;
-    backup_guard.metadata.version = crate::version::Current::new().semver().into();
-    backup_guard.metadata.timestamp = timestamp;
-
-    backup_guard.save_and_unmount().await?;
-
-    ctx.db
-        .mutate(|v| v.as_server_info_mut().as_last_backup_mut().ser(&timestamp))
-        .await?;
-
-    Ok(backup_report)
-}

backend/src/backup/mod.rs

@@ -1,226 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet};
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use chrono::{DateTime, Utc};
-use color_eyre::eyre::eyre;
-use helpers::AtomicFile;
-use models::{ImageId, OptionExt};
-use reqwest::Url;
-use rpc_toolkit::command;
-use serde::{Deserialize, Serialize};
-use tokio::fs::File;
-use tokio::io::AsyncWriteExt;
-use tracing::instrument;
-
-use self::target::PackageBackupInfo;
-use crate::context::RpcContext;
-use crate::install::PKG_ARCHIVE_DIR;
-use crate::manager::manager_seed::ManagerSeed;
-use crate::net::interface::InterfaceId;
-use crate::net::keys::Key;
-use crate::prelude::*;
-use crate::procedure::docker::DockerContainers;
-use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
-use crate::s9pk::manifest::PackageId;
-use crate::util::serde::{Base32, Base64, IoFormat};
-use crate::util::Version;
-use crate::version::{Current, VersionT};
-use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
-use crate::{Error, ErrorKind, ResultExt};
-
-pub mod backup_bulk;
-pub mod os;
-pub mod restore;
-pub mod target;
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct BackupReport {
-    server: ServerBackupReport,
-    packages: BTreeMap<PackageId, PackageBackupReport>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct ServerBackupReport {
-    attempted: bool,
-    error: Option<String>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct PackageBackupReport {
-    pub error: Option<String>,
-}
-
-#[command(subcommands(backup_bulk::backup_all, target::target))]
-pub fn backup() -> Result<(), Error> {
-    Ok(())
-}
-
-#[command(rename = "backup", subcommands(restore::restore_packages_rpc))]
-pub fn package_backup() -> Result<(), Error> {
-    Ok(())
-}
-
-#[derive(Deserialize, Serialize)]
-struct BackupMetadata {
-    pub timestamp: DateTime<Utc>,
-    #[serde(default)]
-    pub network_keys: BTreeMap<InterfaceId, Base64<[u8; 32]>>,
-    #[serde(default)]
-    pub tor_keys: BTreeMap<InterfaceId, Base32<[u8; 64]>>, // DEPRECATED
-    pub marketplace_url: Option<Url>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
-#[model = "Model<Self>"]
-pub struct BackupActions {
-    pub create: PackageProcedure,
-    pub restore: PackageProcedure,
-}
-impl BackupActions {
-    pub fn validate(
-        &self,
-        _container: &Option<DockerContainers>,
-        eos_version: &Version,
-        volumes: &Volumes,
-        image_ids: &BTreeSet<ImageId>,
-    ) -> Result<(), Error> {
-        self.create
-            .validate(eos_version, volumes, image_ids, false)
-            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Create"))?;
-        self.restore
-            .validate(eos_version, volumes, image_ids, false)
-            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Restore"))?;
-        Ok(())
-    }
-
-    #[instrument(skip_all)]
-    pub async fn create(&self, seed: Arc<ManagerSeed>) -> Result<PackageBackupInfo, Error> {
-        let manifest = &seed.manifest;
-        let mut volumes = seed.manifest.volumes.to_readonly();
-        let ctx = &seed.ctx;
-        let pkg_id = &manifest.id;
-        let pkg_version = &manifest.version;
-        volumes.insert(VolumeId::Backup, Volume::Backup { readonly: false });
-        let backup_dir = backup_dir(&manifest.id);
-        if tokio::fs::metadata(&backup_dir).await.is_err() {
-            tokio::fs::create_dir_all(&backup_dir).await?
-        }
-        self.create
-            .execute::<(), NoOutput>(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::CreateBackup,
-                &volumes,
-                None,
-                None,
-            )
-            .await?
-            .map_err(|e| eyre!("{}", e.1))
-            .with_kind(crate::ErrorKind::Backup)?;
-        let (network_keys, tor_keys): (Vec<_>, Vec<_>) =
-            Key::for_package(&ctx.secret_store, pkg_id)
-                .await?
-                .into_iter()
-                .filter_map(|k| {
-                    let interface = k.interface().map(|(_, i)| i)?;
-                    Some((
-                        (interface.clone(), Base64(k.as_bytes())),
-                        (interface, Base32(k.tor_key().as_bytes())),
-                    ))
-                })
-                .unzip();
-        let marketplace_url = ctx
-            .db
-            .peek()
-            .await
-            .as_package_data()
-            .as_idx(&pkg_id)
-            .or_not_found(pkg_id)?
-            .expect_as_installed()?
-            .as_installed()
-            .as_marketplace_url()
-            .de()?;
-        let tmp_path = Path::new(BACKUP_DIR)
-            .join(pkg_id)
-            .join(format!("{}.s9pk", pkg_id));
-        let s9pk_path = ctx
-            .datadir
-            .join(PKG_ARCHIVE_DIR)
-            .join(pkg_id)
-            .join(pkg_version.as_str())
-            .join(format!("{}.s9pk", pkg_id));
-        let mut infile = File::open(&s9pk_path).await?;
-        let mut outfile = AtomicFile::new(&tmp_path, None::<PathBuf>)
-            .await
-            .with_kind(ErrorKind::Filesystem)?;
-        tokio::io::copy(&mut infile, &mut *outfile)
-            .await
-            .with_ctx(|_| {
-                (
-                    crate::ErrorKind::Filesystem,
-                    format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()),
-                )
-            })?;
-        outfile.save().await.with_kind(ErrorKind::Filesystem)?;
-        let timestamp = Utc::now();
-        let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
-        let mut outfile = AtomicFile::new(&metadata_path, None::<PathBuf>)
-            .await
-            .with_kind(ErrorKind::Filesystem)?;
-        let network_keys = network_keys.into_iter().collect();
-        let tor_keys = tor_keys.into_iter().collect();
-        outfile
-            .write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
-                timestamp,
-                network_keys,
-                tor_keys,
-                marketplace_url,
-            })?)
-            .await?;
-        outfile.save().await.with_kind(ErrorKind::Filesystem)?;
-        Ok(PackageBackupInfo {
-            os_version: Current::new().semver().into(),
-            title: manifest.title.clone(),
-            version: pkg_version.clone(),
-            timestamp,
-        })
-    }
-
-    #[instrument(skip_all)]
-    pub async fn restore(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        volumes: &Volumes,
-    ) -> Result<Option<Url>, Error> {
-        let mut volumes = volumes.clone();
-        volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true });
-        self.restore
-            .execute::<(), NoOutput>(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::RestoreBackup,
-                &volumes,
-                None,
-                None,
-            )
-            .await?
-            .map_err(|e| eyre!("{}", e.1))
-            .with_kind(crate::ErrorKind::Restore)?;
-        let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
-        let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
-            &tokio::fs::read(&metadata_path).await.with_ctx(|_| {
-                (
-                    crate::ErrorKind::Filesystem,
-                    metadata_path.display().to_string(),
-                )
-            })?,
-        )?;
-
-        Ok(metadata.marketplace_url)
-    }
-}

backend/src/backup/os.rs

@@ -1,122 +0,0 @@
-use openssl::pkey::PKey;
-use openssl::x509::X509;
-use patch_db::Value;
-use serde::{Deserialize, Serialize};
-
-use crate::account::AccountInfo;
-use crate::hostname::{generate_hostname, generate_id, Hostname};
-use crate::net::keys::Key;
-use crate::prelude::*;
-use crate::util::serde::Base64;
-
-pub struct OsBackup {
-    pub account: AccountInfo,
-    pub ui: Value,
-}
-impl<'de> Deserialize<'de> for OsBackup {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        let tagged = OsBackupSerDe::deserialize(deserializer)?;
-        match tagged.version {
-            0 => patch_db::value::from_value::<OsBackupV0>(tagged.rest)
-                .map_err(serde::de::Error::custom)?
-                .project()
-                .map_err(serde::de::Error::custom),
-            1 => patch_db::value::from_value::<OsBackupV1>(tagged.rest)
-                .map_err(serde::de::Error::custom)?
-                .project()
-                .map_err(serde::de::Error::custom),
-            v => Err(serde::de::Error::custom(&format!(
-                "Unknown backup version {v}"
-            ))),
-        }
-    }
-}
-impl Serialize for OsBackup {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        OsBackupSerDe {
-            version: 1,
-            rest: patch_db::value::to_value(
-                &OsBackupV1::unproject(self).map_err(serde::ser::Error::custom)?,
-            )
-            .map_err(serde::ser::Error::custom)?,
-        }
-        .serialize(serializer)
-    }
-}
-
-#[derive(Deserialize, Serialize)]
-struct OsBackupSerDe {
-    #[serde(default)]
-    version: usize,
-    #[serde(flatten)]
-    rest: Value,
-}
-
-/// V0
-#[derive(Deserialize)]
-#[serde(rename = "kebab-case")]
-struct OsBackupV0 {
-    // tor_key: Base32<[u8; 64]>,
-    root_ca_key: String,  // PEM Encoded OpenSSL Key
-    root_ca_cert: String, // PEM Encoded OpenSSL X509 Certificate
-    ui: Value,            // JSON Value
-}
-impl OsBackupV0 {
-    fn project(self) -> Result<OsBackup, Error> {
-        Ok(OsBackup {
-            account: AccountInfo {
-                server_id: generate_id(),
-                hostname: generate_hostname(),
-                password: Default::default(),
-                key: Key::new(None),
-                root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
-                root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
-            },
-            ui: self.ui,
-        })
-    }
-}
-
-/// V1
-#[derive(Deserialize, Serialize)]
-#[serde(rename = "kebab-case")]
-struct OsBackupV1 {
-    server_id: String,         // uuidv4
-    hostname: String,          // embassy-<adjective>-<noun>
-    net_key: Base64<[u8; 32]>, // Ed25519 Secret Key
-    root_ca_key: String,       // PEM Encoded OpenSSL Key
-    root_ca_cert: String,      // PEM Encoded OpenSSL X509 Certificate
-    ui: Value,                 // JSON Value
-                               // TODO add more
-}
-impl OsBackupV1 {
-    fn project(self) -> Result<OsBackup, Error> {
-        Ok(OsBackup {
-            account: AccountInfo {
-                server_id: self.server_id,
-                hostname: Hostname(self.hostname),
-                password: Default::default(),
-                key: Key::from_bytes(None, self.net_key.0),
-                root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
-                root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
-            },
-            ui: self.ui,
-        })
-    }
-    fn unproject(backup: &OsBackup) -> Result<Self, Error> {
-        Ok(Self {
-            server_id: backup.account.server_id.clone(),
-            hostname: backup.account.hostname.0.clone(),
-            net_key: Base64(backup.account.key.as_bytes()),
-            root_ca_key: String::from_utf8(backup.account.root_ca_key.private_key_to_pem_pkcs8()?)?,
-            root_ca_cert: String::from_utf8(backup.account.root_ca_cert.to_pem()?)?,
-            ui: backup.ui.clone(),
-        })
-    }
-}

backend/src/backup/restore.rs

@@ -1,461 +0,0 @@
-use std::collections::BTreeMap;
-use std::path::Path;
-use std::sync::atomic::Ordering;
-use std::sync::Arc;
-use std::time::Duration;
-
-use clap::ArgMatches;
-use futures::future::BoxFuture;
-use futures::{stream, FutureExt, StreamExt};
-use openssl::x509::X509;
-use rpc_toolkit::command;
-use sqlx::Connection;
-use tokio::fs::File;
-use torut::onion::OnionAddressV3;
-use tracing::instrument;
-
-use super::target::BackupTargetId;
-use crate::backup::os::OsBackup;
-use crate::backup::BackupMetadata;
-use crate::context::rpc::RpcContextConfig;
-use crate::context::{RpcContext, SetupContext};
-use crate::db::model::{PackageDataEntry, PackageDataEntryRestoring, StaticFiles};
-use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
-use crate::disk::mount::filesystem::ReadWrite;
-use crate::disk::mount::guard::TmpMountGuard;
-use crate::hostname::Hostname;
-use crate::init::init;
-use crate::install::progress::InstallProgress;
-use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
-use crate::notifications::NotificationLevel;
-use crate::prelude::*;
-use crate::s9pk::manifest::{Manifest, PackageId};
-use crate::s9pk::reader::S9pkReader;
-use crate::setup::SetupStatus;
-use crate::util::display_none;
-use crate::util::io::dir_size;
-use crate::util::serde::IoFormat;
-use crate::volume::{backup_dir, BACKUP_DIR, PKG_VOLUME_DIR};
-
-fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<PackageId>, Error> {
-    arg.split(',')
-        .map(|s| s.trim().parse().map_err(Error::from))
-        .collect()
-}
-
-#[command(rename = "restore", display(display_none))]
-#[instrument(skip(ctx, password))]
-pub async fn restore_packages_rpc(
-    #[context] ctx: RpcContext,
-    #[arg(parse(parse_comma_separated))] ids: Vec<PackageId>,
-    #[arg(rename = "target-id")] target_id: BackupTargetId,
-    #[arg] password: String,
-) -> Result<(), Error> {
-    let fs = target_id
-        .load(ctx.secret_store.acquire().await?.as_mut())
-        .await?;
-    let backup_guard =
-        BackupMountGuard::mount(TmpMountGuard::mount(&fs, ReadWrite).await?, &password).await?;
-
-    let (backup_guard, tasks, _) = restore_packages(&ctx, backup_guard, ids).await?;
-
-    tokio::spawn(async move {
-        stream::iter(tasks.into_iter().map(|x| (x, ctx.clone())))
-            .for_each_concurrent(5, |(res, ctx)| async move {
-                match res.await {
-                    (Ok(_), _) => (),
-                    (Err(err), package_id) => {
-                        if let Err(err) = ctx
-                            .notification_manager
-                            .notify(
-                                ctx.db.clone(),
-                                Some(package_id.clone()),
-                                NotificationLevel::Error,
-                                "Restoration Failure".to_string(),
-                                format!("Error restoring package {}: {}", package_id, err),
-                                (),
-                                None,
-                            )
-                            .await
-                        {
-                            tracing::error!("Failed to notify: {}", err);
-                            tracing::debug!("{:?}", err);
-                        };
-                        tracing::error!("Error restoring package {}: {}", package_id, err);
-                        tracing::debug!("{:?}", err);
-                    }
-                }
-            })
-            .await;
-        if let Err(e) = backup_guard.unmount().await {
-            tracing::error!("Error unmounting backup drive: {}", e);
-            tracing::debug!("{:?}", e);
-        }
-    });
-
-    Ok(())
-}
-
-async fn approximate_progress(
-    rpc_ctx: &RpcContext,
-    progress: &mut ProgressInfo,
-) -> Result<(), Error> {
-    for (id, size) in &mut progress.target_volume_size {
-        let dir = rpc_ctx.datadir.join(PKG_VOLUME_DIR).join(id).join("data");
-        if tokio::fs::metadata(&dir).await.is_err() {
-            *size = 0;
-        } else {
-            *size = dir_size(&dir, None).await?;
-        }
-    }
-    Ok(())
-}
-
-async fn approximate_progress_loop(
-    ctx: &SetupContext,
-    rpc_ctx: &RpcContext,
-    mut starting_info: ProgressInfo,
-) {
-    loop {
-        if let Err(e) = approximate_progress(rpc_ctx, &mut starting_info).await {
-            tracing::error!("Failed to approximate restore progress: {}", e);
-            tracing::debug!("{:?}", e);
-        } else {
-            *ctx.setup_status.write().await = Some(Ok(starting_info.flatten()));
-        }
-        tokio::time::sleep(Duration::from_secs(1)).await;
-    }
-}
-
-#[derive(Debug, Default)]
-struct ProgressInfo {
-    package_installs: BTreeMap<PackageId, Arc<InstallProgress>>,
-    src_volume_size: BTreeMap<PackageId, u64>,
-    target_volume_size: BTreeMap<PackageId, u64>,
-}
-impl ProgressInfo {
-    fn flatten(&self) -> SetupStatus {
-        let mut total_bytes = 0;
-        let mut bytes_transferred = 0;
-
-        for progress in self.package_installs.values() {
-            total_bytes += ((progress.size.unwrap_or(0) as f64) * 2.2) as u64;
-            bytes_transferred += progress.downloaded.load(Ordering::SeqCst);
-            bytes_transferred += ((progress.validated.load(Ordering::SeqCst) as f64) * 0.2) as u64;
-            bytes_transferred += progress.unpacked.load(Ordering::SeqCst);
-        }
-
-        for size in self.src_volume_size.values() {
-            total_bytes += *size;
-        }
-
-        for size in self.target_volume_size.values() {
-            bytes_transferred += *size;
-        }
-
-        if bytes_transferred > total_bytes {
-            bytes_transferred = total_bytes;
-        }
-
-        SetupStatus {
-            total_bytes: Some(total_bytes),
-            bytes_transferred,
-            complete: false,
-        }
-    }
-}
-
-#[instrument(skip(ctx))]
-pub async fn recover_full_embassy(
-    ctx: SetupContext,
-    disk_guid: Arc<String>,
-    embassy_password: String,
-    recovery_source: TmpMountGuard,
-    recovery_password: Option<String>,
-) -> Result<(Arc<String>, Hostname, OnionAddressV3, X509), Error> {
-    let backup_guard = BackupMountGuard::mount(
-        recovery_source,
-        recovery_password.as_deref().unwrap_or_default(),
-    )
-    .await?;
-
-    let os_backup_path = backup_guard.as_ref().join("os-backup.cbor");
-    let mut os_backup: OsBackup = IoFormat::Cbor.from_slice(
-        &tokio::fs::read(&os_backup_path)
-            .await
-            .with_ctx(|_| (ErrorKind::Filesystem, os_backup_path.display().to_string()))?,
-    )?;
-
-    os_backup.account.password = argon2::hash_encoded(
-        embassy_password.as_bytes(),
-        &rand::random::<[u8; 16]>()[..],
-        &argon2::Config::rfc9106_low_mem(),
-    )
-    .with_kind(ErrorKind::PasswordHashGeneration)?;
-
-    let secret_store = ctx.secret_store().await?;
-
-    os_backup.account.save(&secret_store).await?;
-
-    secret_store.close().await;
-
-    let cfg = RpcContextConfig::load(ctx.config_path.clone()).await?;
-
-    init(&cfg).await?;
-
-    let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid.clone()).await?;
-
-    let ids: Vec<_> = backup_guard
-        .metadata
-        .package_backups
-        .keys()
-        .cloned()
-        .collect();
-    let (backup_guard, tasks, progress_info) =
-        restore_packages(&rpc_ctx, backup_guard, ids).await?;
-    let task_consumer_rpc_ctx = rpc_ctx.clone();
-    tokio::select! {
-        _ = async move {
-            stream::iter(tasks.into_iter().map(|x| (x,  task_consumer_rpc_ctx.clone())))
-                .for_each_concurrent(5, |(res, ctx)| async move {
-                    match res.await {
-                        (Ok(_), _) => (),
-                        (Err(err), package_id) => {
-                            if let Err(err) = ctx.notification_manager.notify(
-                                ctx.db.clone(),
-                                Some(package_id.clone()),
-                                NotificationLevel::Error,
-                                "Restoration Failure".to_string(), format!("Error restoring package {}: {}", package_id,err), (), None).await{
-                                tracing::error!("Failed to notify: {}", err);
-                                tracing::debug!("{:?}", err);
-                                };
-                            tracing::error!("Error restoring package {}: {}", package_id, err);
-                            tracing::debug!("{:?}", err);
-                        },
-                    }
-                }).await;
-
-        } => {
-
-        },
-        _ = approximate_progress_loop(&ctx, &rpc_ctx, progress_info) => unreachable!(concat!(module_path!(), "::approximate_progress_loop should not terminate")),
-    }
-
-    backup_guard.unmount().await?;
-    rpc_ctx.shutdown().await?;
-
-    Ok((
-        disk_guid,
-        os_backup.account.hostname,
-        os_backup.account.key.tor_address(),
-        os_backup.account.root_ca_cert,
-    ))
-}
-
-#[instrument(skip(ctx, backup_guard))]
-async fn restore_packages(
-    ctx: &RpcContext,
-    backup_guard: BackupMountGuard<TmpMountGuard>,
-    ids: Vec<PackageId>,
-) -> Result<
-    (
-        BackupMountGuard<TmpMountGuard>,
-        Vec<BoxFuture<'static, (Result<(), Error>, PackageId)>>,
-        ProgressInfo,
-    ),
-    Error,
-> {
-    let guards = assure_restoring(ctx, ids, &backup_guard).await?;
-
-    let mut progress_info = ProgressInfo::default();
-
-    let mut tasks = Vec::with_capacity(guards.len());
-    for (manifest, guard) in guards {
-        let id = manifest.id.clone();
-        let (progress, task) = restore_package(ctx.clone(), manifest, guard).await?;
-        progress_info
-            .package_installs
-            .insert(id.clone(), progress.clone());
-        progress_info
-            .src_volume_size
-            .insert(id.clone(), dir_size(backup_dir(&id), None).await?);
-        progress_info.target_volume_size.insert(id.clone(), 0);
-        let package_id = id.clone();
-        tasks.push(
-            async move {
-                if let Err(e) = task.await {
-                    tracing::error!("Error restoring package {}: {}", id, e);
-                    tracing::debug!("{:?}", e);
-                    Err(e)
-                } else {
-                    Ok(())
-                }
-            }
-            .map(|x| (x, package_id))
-            .boxed(),
-        );
-    }
-
-    Ok((backup_guard, tasks, progress_info))
-}
-
-#[instrument(skip(ctx, backup_guard))]
-async fn assure_restoring(
-    ctx: &RpcContext,
-    ids: Vec<PackageId>,
-    backup_guard: &BackupMountGuard<TmpMountGuard>,
-) -> Result<Vec<(Manifest, PackageBackupMountGuard)>, Error> {
-    let mut guards = Vec::with_capacity(ids.len());
-
-    let mut insert_packages = BTreeMap::new();
-
-    for id in ids {
-        let peek = ctx.db.peek().await;
-
-        let model = peek.as_package_data().as_idx(&id);
-
-        if !model.is_none() {
-            return Err(Error::new(
-                eyre!("Can't restore over existing package: {}", id),
-                crate::ErrorKind::InvalidRequest,
-            ));
-        }
-        let guard = backup_guard.mount_package_backup(&id).await?;
-        let s9pk_path = Path::new(BACKUP_DIR).join(&id).join(format!("{}.s9pk", id));
-        let mut rdr = S9pkReader::open(&s9pk_path, false).await?;
-
-        let manifest = rdr.manifest().await?;
-        let version = manifest.version.clone();
-        let progress = Arc::new(InstallProgress::new(Some(
-            tokio::fs::metadata(&s9pk_path).await?.len(),
-        )));
-
-        let public_dir_path = ctx
-            .datadir
-            .join(PKG_PUBLIC_DIR)
-            .join(&id)
-            .join(version.as_str());
-        tokio::fs::create_dir_all(&public_dir_path).await?;
-
-        let license_path = public_dir_path.join("LICENSE.md");
-        let mut dst = File::create(&license_path).await?;
-        tokio::io::copy(&mut rdr.license().await?, &mut dst).await?;
-        dst.sync_all().await?;
-
-        let instructions_path = public_dir_path.join("INSTRUCTIONS.md");
-        let mut dst = File::create(&instructions_path).await?;
-        tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?;
-        dst.sync_all().await?;
-
-        let icon_path = Path::new("icon").with_extension(&manifest.assets.icon_type());
-        let icon_path = public_dir_path.join(&icon_path);
-        let mut dst = File::create(&icon_path).await?;
-        tokio::io::copy(&mut rdr.icon().await?, &mut dst).await?;
-        dst.sync_all().await?;
-        insert_packages.insert(
-            id.clone(),
-            PackageDataEntry::Restoring(PackageDataEntryRestoring {
-                install_progress: progress.clone(),
-                static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()),
-                manifest: manifest.clone(),
-            }),
-        );
-
-        guards.push((manifest, guard));
-    }
-    ctx.db
-        .mutate(|db| {
-            for (id, package) in insert_packages {
-                db.as_package_data_mut().insert(&id, &package)?;
-            }
-            Ok(())
-        })
-        .await?;
-    Ok(guards)
-}
-
-#[instrument(skip(ctx, guard))]
-async fn restore_package<'a>(
-    ctx: RpcContext,
-    manifest: Manifest,
-    guard: PackageBackupMountGuard,
-) -> Result<(Arc<InstallProgress>, BoxFuture<'static, Result<(), Error>>), Error> {
-    let id = manifest.id.clone();
-    let s9pk_path = Path::new(BACKUP_DIR)
-        .join(&manifest.id)
-        .join(format!("{}.s9pk", id));
-
-    let metadata_path = Path::new(BACKUP_DIR).join(&id).join("metadata.cbor");
-    let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
-        &tokio::fs::read(&metadata_path)
-            .await
-            .with_ctx(|_| (ErrorKind::Filesystem, metadata_path.display().to_string()))?,
-    )?;
-
-    let mut secrets = ctx.secret_store.acquire().await?;
-    let mut secrets_tx = secrets.begin().await?;
-    for (iface, key) in metadata.network_keys {
-        let k = key.0.as_slice();
-        sqlx::query!(
-            "INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
-            id.to_string(),
-            iface.to_string(),
-            k,
-        )
-        .execute(secrets_tx.as_mut()).await?;
-    }
-    // DEPRECATED
-    for (iface, key) in metadata.tor_keys {
-        let k = key.0.as_slice();
-        sqlx::query!(
-            "INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
-            id.to_string(),
-            iface.to_string(),
-            k,
-        )
-        .execute(secrets_tx.as_mut()).await?;
-    }
-    secrets_tx.commit().await?;
-    drop(secrets);
-
-    let len = tokio::fs::metadata(&s9pk_path)
-        .await
-        .with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?
-        .len();
-    let file = File::open(&s9pk_path)
-        .await
-        .with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?;
-
-    let progress = InstallProgress::new(Some(len));
-    let marketplace_url = metadata.marketplace_url;
-
-    let progress = Arc::new(progress);
-
-    ctx.db
-        .mutate(|db| {
-            db.as_package_data_mut().insert(
-                &id,
-                &PackageDataEntry::Restoring(PackageDataEntryRestoring {
-                    install_progress: progress.clone(),
-                    static_files: StaticFiles::local(
-                        &id,
-                        &manifest.version,
-                        manifest.assets.icon_type(),
-                    ),
-                    manifest: manifest.clone(),
-                }),
-            )
-        })
-        .await?;
-    Ok((
-        progress.clone(),
-        async move {
-            download_install_s9pk(ctx, manifest, marketplace_url, progress, file, None).await?;
-
-            guard.unmount().await?;
-
-            Ok(())
-        }
-        .boxed(),
-    ))
-}

backend/src/backup/target/cifs.rs

@@ -1,211 +0,0 @@
-use std::path::{Path, PathBuf};
-
-use color_eyre::eyre::eyre;
-use futures::TryStreamExt;
-use rpc_toolkit::command;
-use serde::{Deserialize, Serialize};
-use sqlx::{Executor, Postgres};
-
-use super::{BackupTarget, BackupTargetId};
-use crate::context::RpcContext;
-use crate::disk::mount::filesystem::cifs::Cifs;
-use crate::disk::mount::filesystem::ReadOnly;
-use crate::disk::mount::guard::TmpMountGuard;
-use crate::disk::util::{recovery_info, EmbassyOsRecoveryInfo};
-use crate::prelude::*;
-use crate::util::display_none;
-use crate::util::serde::KeyVal;
-
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct CifsBackupTarget {
-    hostname: String,
-    path: PathBuf,
-    username: String,
-    mountable: bool,
-    embassy_os: Option<EmbassyOsRecoveryInfo>,
-}
-
-#[command(subcommands(add, update, remove))]
-pub fn cifs() -> Result<(), Error> {
-    Ok(())
-}
-
-#[command(display(display_none))]
-pub async fn add(
-    #[context] ctx: RpcContext,
-    #[arg] hostname: String,
-    #[arg] path: PathBuf,
-    #[arg] username: String,
-    #[arg] password: Option<String>,
-) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
-    let cifs = Cifs {
-        hostname,
-        path,
-        username,
-        password,
-    };
-    let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
-    let embassy_os = recovery_info(&guard).await?;
-    guard.unmount().await?;
-    let path_string = Path::new("/").join(&cifs.path).display().to_string();
-    let id: i32 = sqlx::query!(
-        "INSERT INTO cifs_shares (hostname, path, username, password) VALUES ($1, $2, $3, $4) RETURNING id",
-        cifs.hostname,
-        path_string,
-        cifs.username,
-        cifs.password,
-    )
-    .fetch_one(&ctx.secret_store)
-    .await?.id;
-    Ok(KeyVal {
-        key: BackupTargetId::Cifs { id },
-        value: BackupTarget::Cifs(CifsBackupTarget {
-            hostname: cifs.hostname,
-            path: cifs.path,
-            username: cifs.username,
-            mountable: true,
-            embassy_os,
-        }),
-    })
-}
-
-#[command(display(display_none))]
-pub async fn update(
-    #[context] ctx: RpcContext,
-    #[arg] id: BackupTargetId,
-    #[arg] hostname: String,
-    #[arg] path: PathBuf,
-    #[arg] username: String,
-    #[arg] password: Option<String>,
-) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
-    let id = if let BackupTargetId::Cifs { id } = id {
-        id
-    } else {
-        return Err(Error::new(
-            eyre!("Backup Target ID {} Not Found", id),
-            ErrorKind::NotFound,
-        ));
-    };
-    let cifs = Cifs {
-        hostname,
-        path,
-        username,
-        password,
-    };
-    let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
-    let embassy_os = recovery_info(&guard).await?;
-    guard.unmount().await?;
-    let path_string = Path::new("/").join(&cifs.path).display().to_string();
-    if sqlx::query!(
-        "UPDATE cifs_shares SET hostname = $1, path = $2, username = $3, password = $4 WHERE id = $5",
-        cifs.hostname,
-        path_string,
-        cifs.username,
-        cifs.password,
-        id,
-    )
-    .execute(&ctx.secret_store)
-    .await?
-    .rows_affected()
-        == 0
-    {
-        return Err(Error::new(
-            eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
-            ErrorKind::NotFound,
-        ));
-    };
-    Ok(KeyVal {
-        key: BackupTargetId::Cifs { id },
-        value: BackupTarget::Cifs(CifsBackupTarget {
-            hostname: cifs.hostname,
-            path: cifs.path,
-            username: cifs.username,
-            mountable: true,
-            embassy_os,
-        }),
-    })
-}
-
-#[command(display(display_none))]
-pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Result<(), Error> {
-    let id = if let BackupTargetId::Cifs { id } = id {
-        id
-    } else {
-        return Err(Error::new(
-            eyre!("Backup Target ID {} Not Found", id),
-            ErrorKind::NotFound,
-        ));
-    };
-    if sqlx::query!("DELETE FROM cifs_shares WHERE id = $1", id)
-        .execute(&ctx.secret_store)
-        .await?
-        .rows_affected()
-        == 0
-    {
-        return Err(Error::new(
-            eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
-            ErrorKind::NotFound,
-        ));
-    };
-    Ok(())
-}
-
-pub async fn load<Ex>(secrets: &mut Ex, id: i32) -> Result<Cifs, Error>
-where
-    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-{
-    let record = sqlx::query!(
-        "SELECT hostname, path, username, password FROM cifs_shares WHERE id = $1",
-        id
-    )
-    .fetch_one(secrets)
-    .await?;
-
-    Ok(Cifs {
-        hostname: record.hostname,
-        path: PathBuf::from(record.path),
-        username: record.username,
-        password: record.password,
-    })
-}
-
-pub async fn list<Ex>(secrets: &mut Ex) -> Result<Vec<(i32, CifsBackupTarget)>, Error>
-where
-    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-{
-    let mut records =
-        sqlx::query!("SELECT id, hostname, path, username, password FROM cifs_shares")
-            .fetch_many(secrets);
-
-    let mut cifs = Vec::new();
-    while let Some(query_result) = records.try_next().await? {
-        if let Some(record) = query_result.right() {
-            let mount_info = Cifs {
-                hostname: record.hostname,
-                path: PathBuf::from(record.path),
-                username: record.username,
-                password: record.password,
-            };
-            let embassy_os = async {
-                let guard = TmpMountGuard::mount(&mount_info, ReadOnly).await?;
-                let embassy_os = recovery_info(&guard).await?;
-                guard.unmount().await?;
-                Ok::<_, Error>(embassy_os)
-            }
-            .await;
-            cifs.push((
-                record.id,
-                CifsBackupTarget {
-                    hostname: mount_info.hostname,
-                    path: mount_info.path,
-                    username: mount_info.username,
-                    mountable: embassy_os.is_ok(),
-                    embassy_os: embassy_os.ok().and_then(|a| a),
-                },
-            ));
-        }
-    }
-
-    Ok(cifs)
-}

backend/src/bins/avahi_alias.rs

@@ -1,163 +0,0 @@
-use avahi_sys::{
-    self, avahi_client_errno, avahi_entry_group_add_service, avahi_entry_group_commit,
-    avahi_strerror, AvahiClient,
-};
-
-fn log_str_error(action: &str, e: i32) {
-    unsafe {
-        let e_str = avahi_strerror(e);
-        eprintln!(
-            "Could not {}: {:?}",
-            action,
-            std::ffi::CStr::from_ptr(e_str)
-        );
-    }
-}
-
-pub fn main() {
-    let aliases: Vec<_> = std::env::args().skip(1).collect();
-    unsafe {
-        let simple_poll = avahi_sys::avahi_simple_poll_new();
-        let poll = avahi_sys::avahi_simple_poll_get(simple_poll);
-        let mut box_err = Box::pin(0 as i32);
-        let err_c: *mut i32 = box_err.as_mut().get_mut();
-        let avahi_client = avahi_sys::avahi_client_new(
-            poll,
-            avahi_sys::AvahiClientFlags::AVAHI_CLIENT_NO_FAIL,
-            Some(client_callback),
-            std::ptr::null_mut(),
-            err_c,
-        );
-        if avahi_client == std::ptr::null_mut::<AvahiClient>() {
-            log_str_error("create Avahi client", *box_err);
-            panic!("Failed to create Avahi Client");
-        }
-        let group = avahi_sys::avahi_entry_group_new(
-            avahi_client,
-            Some(entry_group_callback),
-            std::ptr::null_mut(),
-        );
-        if group == std::ptr::null_mut() {
-            log_str_error("create Avahi entry group", avahi_client_errno(avahi_client));
-            panic!("Failed to create Avahi Entry Group");
-        }
-        let mut hostname_buf = vec![0];
-        let hostname_raw = avahi_sys::avahi_client_get_host_name_fqdn(avahi_client);
-        hostname_buf.extend_from_slice(std::ffi::CStr::from_ptr(hostname_raw).to_bytes_with_nul());
-        let buflen = hostname_buf.len();
-        debug_assert!(hostname_buf.ends_with(b".local\0"));
-        debug_assert!(!hostname_buf[..(buflen - 7)].contains(&b'.'));
-        // assume fixed length prefix on hostname due to local address
-        hostname_buf[0] = (buflen - 8) as u8; // set the prefix length to len - 8 (leading byte, .local, nul) for the main address
-        hostname_buf[buflen - 7] = 5; // set the prefix length to 5 for "local"
-        let mut res;
-        let http_tcp_cstr =
-            std::ffi::CString::new("_http._tcp").expect("Could not cast _http._tcp to c string");
-        res = avahi_entry_group_add_service(
-            group,
-            avahi_sys::AVAHI_IF_UNSPEC,
-            avahi_sys::AVAHI_PROTO_UNSPEC,
-            avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST,
-            hostname_raw,
-            http_tcp_cstr.as_ptr(),
-            std::ptr::null(),
-            std::ptr::null(),
-            443,
-            // below is a secret final argument that the type signature of this function does not tell you that it
-            // needs. This is because the C lib function takes a variable number of final arguments indicating the
-            // desired TXT records to add to this service entry. The way it decides when to stop taking arguments
-            // from the stack and dereferencing them is when it finds a null pointer...because fuck you, that's why.
-            // The consequence of this is that forgetting this last argument will cause segfaults or other undefined
-            // behavior. Welcome back to the stone age motherfucker.
-            std::ptr::null::<libc::c_char>(),
-        );
-        if res < avahi_sys::AVAHI_OK {
-            log_str_error("add service to Avahi entry group", res);
-            panic!("Failed to load Avahi services");
-        }
-        eprintln!("Published {:?}", std::ffi::CStr::from_ptr(hostname_raw));
-        for alias in aliases {
-            let lan_address = alias + ".local";
-            let lan_address_ptr = std::ffi::CString::new(lan_address)
-                .expect("Could not cast lan address to c string");
-            res = avahi_sys::avahi_entry_group_add_record(
-                group,
-                avahi_sys::AVAHI_IF_UNSPEC,
-                avahi_sys::AVAHI_PROTO_UNSPEC,
-                avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST
-                    | avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_ALLOW_MULTIPLE,
-                lan_address_ptr.as_ptr(),
-                avahi_sys::AVAHI_DNS_CLASS_IN as u16,
-                avahi_sys::AVAHI_DNS_TYPE_CNAME as u16,
-                avahi_sys::AVAHI_DEFAULT_TTL,
-                hostname_buf.as_ptr().cast(),
-                hostname_buf.len(),
-            );
-            if res < avahi_sys::AVAHI_OK {
-                log_str_error("add CNAME record to Avahi entry group", res);
-                panic!("Failed to load Avahi services");
-            }
-            eprintln!("Published {:?}", lan_address_ptr);
-        }
-        let commit_err = avahi_entry_group_commit(group);
-        if commit_err < avahi_sys::AVAHI_OK {
-            log_str_error("reset Avahi entry group", commit_err);
-            panic!("Failed to load Avahi services: reset");
-        }
-    }
-    std::thread::park()
-}
-
-unsafe extern "C" fn entry_group_callback(
-    _group: *mut avahi_sys::AvahiEntryGroup,
-    state: avahi_sys::AvahiEntryGroupState,
-    _userdata: *mut core::ffi::c_void,
-) {
-    match state {
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_FAILURE => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_FAILURE");
-        }
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_COLLISION => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_COLLISION");
-        }
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_UNCOMMITED => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_UNCOMMITED");
-        }
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_ESTABLISHED => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_ESTABLISHED");
-        }
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_REGISTERING => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_REGISTERING");
-        }
-        other => {
-            eprintln!("AvahiCallback: EntryGroupState = {}", other);
-        }
-    }
-}
-
-unsafe extern "C" fn client_callback(
-    _group: *mut avahi_sys::AvahiClient,
-    state: avahi_sys::AvahiClientState,
-    _userdata: *mut core::ffi::c_void,
-) {
-    match state {
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_FAILURE => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_FAILURE");
-        }
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_RUNNING => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_RUNNING");
-        }
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_CONNECTING => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_CONNECTING");
-        }
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_COLLISION => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_COLLISION");
-        }
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_REGISTERING => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_REGISTERING");
-        }
-        other => {
-            eprintln!("AvahiCallback: ClientState = {}", other);
-        }
-    }
-}

backend/src/bins/mod.rs

@@ -1,59 +0,0 @@
-use std::path::Path;
-
-#[cfg(feature = "avahi-alias")]
-pub mod avahi_alias;
-pub mod deprecated;
-#[cfg(feature = "cli")]
-pub mod start_cli;
-#[cfg(feature = "js_engine")]
-pub mod start_deno;
-#[cfg(feature = "daemon")]
-pub mod start_init;
-#[cfg(feature = "sdk")]
-pub mod start_sdk;
-#[cfg(feature = "daemon")]
-pub mod startd;
-
-fn select_executable(name: &str) -> Option<fn()> {
-    match name {
-        #[cfg(feature = "avahi-alias")]
-        "avahi-alias" => Some(avahi_alias::main),
-        #[cfg(feature = "js_engine")]
-        "start-deno" => Some(start_deno::main),
-        #[cfg(feature = "cli")]
-        "start-cli" => Some(start_cli::main),
-        #[cfg(feature = "sdk")]
-        "start-sdk" => Some(start_sdk::main),
-        #[cfg(feature = "daemon")]
-        "startd" => Some(startd::main),
-        "embassy-cli" => Some(|| deprecated::renamed("embassy-cli", "start-cli")),
-        "embassy-sdk" => Some(|| deprecated::renamed("embassy-sdk", "start-sdk")),
-        "embassyd" => Some(|| deprecated::renamed("embassyd", "startd")),
-        "embassy-init" => Some(|| deprecated::removed("embassy-init")),
-        _ => None,
-    }
-}
-
-pub fn startbox() {
-    let args = std::env::args().take(2).collect::<Vec<_>>();
-    if let Some(x) = args
-        .get(0)
-        .and_then(|s| Path::new(&*s).file_name())
-        .and_then(|s| s.to_str())
-        .and_then(|s| select_executable(&s))
-    {
-        x()
-    } else if let Some(x) = args.get(1).and_then(|s| select_executable(&s)) {
-        x()
-    } else {
-        eprintln!(
-            "unknown executable: {}",
-            args.get(0)
-                .filter(|x| &**x != "startbox")
-                .or_else(|| args.get(1))
-                .map(|s| s.as_str())
-                .unwrap_or("N/A")
-        );
-        std::process::exit(1);
-    }
-}

backend/src/bins/start_cli.rs

@@ -1,62 +0,0 @@
-use clap::Arg;
-use rpc_toolkit::run_cli;
-use rpc_toolkit::yajrc::RpcError;
-use serde_json::Value;
-
-use crate::context::CliContext;
-use crate::util::logger::EmbassyLogger;
-use crate::version::{Current, VersionT};
-use crate::Error;
-
-lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
-}
-
-fn inner_main() -> Result<(), Error> {
-    run_cli!({
-        command: crate::main_api,
-        app: app => app
-            .name("StartOS CLI")
-            .version(&**VERSION_STRING)
-            .arg(
-                clap::Arg::with_name("config")
-                    .short('c')
-                    .long("config")
-                    .takes_value(true),
-            )
-            .arg(Arg::with_name("host").long("host").short('h').takes_value(true))
-            .arg(Arg::with_name("proxy").long("proxy").short('p').takes_value(true)),
-        context: matches => {
-            EmbassyLogger::init();
-            CliContext::init(matches)?
-        },
-        exit: |e: RpcError| {
-            match e.data {
-                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
-                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
-                    eprintln!("{}: {}", e.message, s);
-                    if let Some(Value::String(s)) = o.get("debug") {
-                        tracing::debug!("{}", s)
-                    }
-                }
-                Some(a) => eprintln!("{}: {}", e.message, a),
-                None => eprintln!("{}", e.message),
-            }
-
-            std::process::exit(e.code);
-        }
-    });
-    Ok(())
-}
-
-pub fn main() {
-    match inner_main() {
-        Ok(_) => (),
-        Err(e) => {
-            eprintln!("{}", e.source);
-            tracing::debug!("{:?}", e.source);
-            drop(e.source);
-            std::process::exit(e.kind as i32)
-        }
-    }
-}

backend/src/bins/start_deno.rs

@@ -1,134 +0,0 @@
-use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::{command, run_cli, Context};
-use serde_json::Value;
-
-use crate::procedure::js_scripts::ExecuteArgs;
-use crate::s9pk::manifest::PackageId;
-use crate::util::serde::{display_serializable, parse_stdin_deserializable};
-use crate::version::{Current, VersionT};
-use crate::Error;
-
-lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
-}
-
-struct DenoContext;
-impl Context for DenoContext {}
-
-#[command(subcommands(execute, sandbox))]
-fn deno_api() -> Result<(), Error> {
-    Ok(())
-}
-
-#[command(cli_only, display(display_serializable))]
-async fn execute(
-    #[arg(stdin, parse(parse_stdin_deserializable))] arg: ExecuteArgs,
-) -> Result<Result<Value, (i32, String)>, Error> {
-    let ExecuteArgs {
-        procedure,
-        directory,
-        pkg_id,
-        pkg_version,
-        name,
-        volumes,
-        input,
-    } = arg;
-    PackageLogger::init(&pkg_id);
-    procedure
-        .execute_impl(&directory, &pkg_id, &pkg_version, name, &volumes, input)
-        .await
-}
-#[command(cli_only, display(display_serializable))]
-async fn sandbox(
-    #[arg(stdin, parse(parse_stdin_deserializable))] arg: ExecuteArgs,
-) -> Result<Result<Value, (i32, String)>, Error> {
-    let ExecuteArgs {
-        procedure,
-        directory,
-        pkg_id,
-        pkg_version,
-        name,
-        volumes,
-        input,
-    } = arg;
-    PackageLogger::init(&pkg_id);
-    procedure
-        .sandboxed_impl(&directory, &pkg_id, &pkg_version, &volumes, input, name)
-        .await
-}
-
-use tracing::Subscriber;
-use tracing_subscriber::util::SubscriberInitExt;
-
-#[derive(Clone)]
-struct PackageLogger {}
-
-impl PackageLogger {
-    fn base_subscriber(id: &PackageId) -> impl Subscriber {
-        use tracing_error::ErrorLayer;
-        use tracing_subscriber::prelude::*;
-        use tracing_subscriber::{fmt, EnvFilter};
-
-        let filter_layer = EnvFilter::default().add_directive(
-            format!("{}=warn", std::module_path!().split("::").next().unwrap())
-                .parse()
-                .unwrap(),
-        );
-        let fmt_layer = fmt::layer().with_writer(std::io::stderr).with_target(true);
-        let journald_layer = tracing_journald::layer()
-            .unwrap()
-            .with_syslog_identifier(format!("{id}.embassy"));
-
-        let sub = tracing_subscriber::registry()
-            .with(filter_layer)
-            .with(fmt_layer)
-            .with(journald_layer)
-            .with(ErrorLayer::default());
-
-        sub
-    }
-    pub fn init(id: &PackageId) -> Self {
-        Self::base_subscriber(id).init();
-        color_eyre::install().unwrap_or_else(|_| tracing::warn!("tracing too many times"));
-
-        Self {}
-    }
-}
-
-fn inner_main() -> Result<(), Error> {
-    run_cli!({
-        command: deno_api,
-        app: app => app
-            .name("StartOS Deno Executor")
-            .version(&**VERSION_STRING),
-        context: _m => DenoContext,
-        exit: |e: RpcError| {
-            match e.data {
-                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
-                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
-                    eprintln!("{}: {}", e.message, s);
-                    if let Some(Value::String(s)) = o.get("debug") {
-                        tracing::debug!("{}", s)
-                    }
-                }
-                Some(a) => eprintln!("{}: {}", e.message, a),
-                None => eprintln!("{}", e.message),
-            }
-
-            std::process::exit(e.code);
-        }
-    });
-    Ok(())
-}
-
-pub fn main() {
-    match inner_main() {
-        Ok(_) => (),
-        Err(e) => {
-            eprintln!("{}", e.source);
-            tracing::debug!("{:?}", e.source);
-            drop(e.source);
-            std::process::exit(e.kind as i32)
-        }
-    }
-}

backend/src/bins/start_init.rs

@@ -1,268 +0,0 @@
-use std::net::{Ipv6Addr, SocketAddr};
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::time::Duration;
-
-use tokio::process::Command;
-use tracing::instrument;
-
-use crate::context::rpc::RpcContextConfig;
-use crate::context::{DiagnosticContext, InstallContext, SetupContext};
-use crate::disk::fsck::RepairStrategy;
-use crate::disk::main::DEFAULT_PASSWORD;
-use crate::disk::REPAIR_DISK_PATH;
-use crate::firmware::update_firmware;
-use crate::init::STANDBY_MODE_PATH;
-use crate::net::web_server::WebServer;
-use crate::shutdown::Shutdown;
-use crate::sound::CHIME;
-use crate::util::Invoke;
-use crate::{Error, ErrorKind, ResultExt, PLATFORM};
-
-#[instrument(skip_all)]
-async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
-    if update_firmware().await?.0 {
-        return Ok(Some(Shutdown {
-            export_args: None,
-            restart: true,
-        }));
-    }
-
-    Command::new("ln")
-        .arg("-sf")
-        .arg("/usr/lib/startos/scripts/fake-apt")
-        .arg("/usr/local/bin/apt")
-        .invoke(crate::ErrorKind::OpenSsh)
-        .await?;
-    Command::new("ln")
-        .arg("-sf")
-        .arg("/usr/lib/startos/scripts/fake-apt")
-        .arg("/usr/local/bin/apt-get")
-        .invoke(crate::ErrorKind::OpenSsh)
-        .await?;
-    Command::new("ln")
-        .arg("-sf")
-        .arg("/usr/lib/startos/scripts/fake-apt")
-        .arg("/usr/local/bin/aptitude")
-        .invoke(crate::ErrorKind::OpenSsh)
-        .await?;
-
-    Command::new("make-ssl-cert")
-        .arg("generate-default-snakeoil")
-        .arg("--force-overwrite")
-        .invoke(crate::ErrorKind::OpenSsl)
-        .await?;
-
-    if tokio::fs::metadata("/run/live/medium").await.is_ok() {
-        Command::new("sed")
-            .arg("-i")
-            .arg("s/PasswordAuthentication no/PasswordAuthentication yes/g")
-            .arg("/etc/ssh/sshd_config")
-            .invoke(crate::ErrorKind::Filesystem)
-            .await?;
-        Command::new("systemctl")
-            .arg("reload")
-            .arg("ssh")
-            .invoke(crate::ErrorKind::OpenSsh)
-            .await?;
-
-        let ctx = InstallContext::init(cfg_path).await?;
-
-        let server = WebServer::install(
-            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
-            ctx.clone(),
-        )
-        .await?;
-
-        tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
-        CHIME.play().await?;
-
-        ctx.shutdown
-            .subscribe()
-            .recv()
-            .await
-            .expect("context dropped");
-
-        server.shutdown().await;
-
-        Command::new("reboot")
-            .invoke(crate::ErrorKind::Unknown)
-            .await?;
-    } else if tokio::fs::metadata("/media/embassy/config/disk.guid")
-        .await
-        .is_err()
-    {
-        let ctx = SetupContext::init(cfg_path).await?;
-
-        let server = WebServer::setup(
-            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
-            ctx.clone(),
-        )
-        .await?;
-
-        tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
-        CHIME.play().await?;
-        ctx.shutdown
-            .subscribe()
-            .recv()
-            .await
-            .expect("context dropped");
-
-        server.shutdown().await;
-
-        tokio::task::yield_now().await;
-        if let Err(e) = Command::new("killall")
-            .arg("firefox-esr")
-            .invoke(ErrorKind::NotFound)
-            .await
-        {
-            tracing::error!("Failed to kill kiosk: {}", e);
-            tracing::debug!("{:?}", e);
-        }
-    } else {
-        let cfg = RpcContextConfig::load(cfg_path).await?;
-        let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
-            .await?;
-        let guid = guid_string.trim();
-        let requires_reboot = crate::disk::main::import(
-            guid,
-            cfg.datadir(),
-            if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
-                RepairStrategy::Aggressive
-            } else {
-                RepairStrategy::Preen
-            },
-            if guid.ends_with("_UNENC") {
-                None
-            } else {
-                Some(DEFAULT_PASSWORD)
-            },
-        )
-        .await?;
-        if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
-            tokio::fs::remove_file(REPAIR_DISK_PATH)
-                .await
-                .with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
-        }
-        if requires_reboot.0 {
-            crate::disk::main::export(guid, cfg.datadir()).await?;
-            Command::new("reboot")
-                .invoke(crate::ErrorKind::Unknown)
-                .await?;
-        }
-        tracing::info!("Loaded Disk");
-        crate::init::init(&cfg).await?;
-    }
-
-    Ok(None)
-}
-
-async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
-    let script = path.as_ref();
-    if script.exists() {
-        match Command::new("/bin/bash").arg(script).spawn() {
-            Ok(mut c) => {
-                if let Err(e) = c.wait().await {
-                    tracing::error!("Error Running {}: {}", script.display(), e);
-                    tracing::debug!("{:?}", e);
-                }
-            }
-            Err(e) => {
-                tracing::error!("Error Running {}: {}", script.display(), e);
-                tracing::debug!("{:?}", e);
-            }
-        }
-    }
-}
-
-#[instrument(skip_all)]
-async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
-    if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
-        tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
-        Command::new("sync").invoke(ErrorKind::Filesystem).await?;
-        crate::sound::SHUTDOWN.play().await?;
-        futures::future::pending::<()>().await;
-    }
-
-    crate::sound::BEP.play().await?;
-
-    run_script_if_exists("/media/embassy/config/preinit.sh").await;
-
-    let res = match setup_or_init(cfg_path.clone()).await {
-        Err(e) => {
-            async move {
-                tracing::error!("{}", e.source);
-                tracing::debug!("{}", e.source);
-                crate::sound::BEETHOVEN.play().await?;
-
-                let ctx = DiagnosticContext::init(
-                    cfg_path,
-                    if tokio::fs::metadata("/media/embassy/config/disk.guid")
-                        .await
-                        .is_ok()
-                    {
-                        Some(Arc::new(
-                            tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
-                                .await?
-                                .trim()
-                                .to_owned(),
-                        ))
-                    } else {
-                        None
-                    },
-                    e,
-                )
-                .await?;
-
-                let server = WebServer::diagnostic(
-                    SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
-                    ctx.clone(),
-                )
-                .await?;
-
-                let shutdown = ctx.shutdown.subscribe().recv().await.unwrap();
-
-                server.shutdown().await;
-
-                Ok(shutdown)
-            }
-            .await
-        }
-        Ok(s) => Ok(s),
-    };
-
-    run_script_if_exists("/media/embassy/config/postinit.sh").await;
-
-    res
-}
-
-pub fn main() {
-    let matches = clap::App::new("start-init")
-        .arg(
-            clap::Arg::with_name("config")
-                .short('c')
-                .long("config")
-                .takes_value(true),
-        )
-        .get_matches();
-
-    let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
-    let res = {
-        let rt = tokio::runtime::Builder::new_multi_thread()
-            .enable_all()
-            .build()
-            .expect("failed to initialize runtime");
-        rt.block_on(inner_main(cfg_path))
-    };
-
-    match res {
-        Ok(Some(shutdown)) => shutdown.execute(),
-        Ok(None) => (),
-        Err(e) => {
-            eprintln!("{}", e.source);
-            tracing::debug!("{:?}", e.source);
-            drop(e.source);
-            std::process::exit(e.kind as i32)
-        }
-    }
-}

backend/src/bins/start_sdk.rs

@@ -1,61 +0,0 @@
-use rpc_toolkit::run_cli;
-use rpc_toolkit::yajrc::RpcError;
-use serde_json::Value;
-
-use crate::context::SdkContext;
-use crate::util::logger::EmbassyLogger;
-use crate::version::{Current, VersionT};
-use crate::Error;
-
-lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
-}
-
-fn inner_main() -> Result<(), Error> {
-    run_cli!({
-        command: crate::portable_api,
-        app: app => app
-            .name("StartOS SDK")
-            .version(&**VERSION_STRING)
-            .arg(
-                clap::Arg::with_name("config")
-                    .short('c')
-                    .long("config")
-                    .takes_value(true),
-            ),
-        context: matches => {
-            if let Err(_) = std::env::var("RUST_LOG") {
-                std::env::set_var("RUST_LOG", "embassy=warn,js_engine=warn");
-            }
-            EmbassyLogger::init();
-            SdkContext::init(matches)?
-        },
-        exit: |e: RpcError| {
-            match e.data {
-                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
-                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
-                    eprintln!("{}: {}", e.message, s);
-                    if let Some(Value::String(s)) = o.get("debug") {
-                        tracing::debug!("{}", s)
-                    }
-                }
-                Some(a) => eprintln!("{}: {}", e.message, a),
-                None => eprintln!("{}", e.message),
-            }
-            std::process::exit(e.code);
-        }
-    });
-    Ok(())
-}
-
-pub fn main() {
-    match inner_main() {
-        Ok(_) => (),
-        Err(e) => {
-            eprintln!("{}", e.source);
-            tracing::debug!("{:?}", e.source);
-            drop(e.source);
-            std::process::exit(e.kind as i32)
-        }
-    }
-}

backend/src/config/action.rs

@@ -1,116 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet};
-
-use color_eyre::eyre::eyre;
-use models::ImageId;
-use patch_db::HasModel;
-use serde::{Deserialize, Serialize};
-use tracing::instrument;
-
-use super::{Config, ConfigSpec};
-use crate::context::RpcContext;
-use crate::dependencies::Dependencies;
-use crate::prelude::*;
-use crate::procedure::docker::DockerContainers;
-use crate::procedure::{PackageProcedure, ProcedureName};
-use crate::s9pk::manifest::PackageId;
-use crate::status::health_check::HealthCheckId;
-use crate::util::Version;
-use crate::volume::Volumes;
-use crate::{Error, ResultExt};
-
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct ConfigRes {
-    pub config: Option<Config>,
-    pub spec: ConfigSpec,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
-#[model = "Model<Self>"]
-pub struct ConfigActions {
-    pub get: PackageProcedure,
-    pub set: PackageProcedure,
-}
-impl ConfigActions {
-    #[instrument(skip_all)]
-    pub fn validate(
-        &self,
-        _container: &Option<DockerContainers>,
-        eos_version: &Version,
-        volumes: &Volumes,
-        image_ids: &BTreeSet<ImageId>,
-    ) -> Result<(), Error> {
-        self.get
-            .validate(eos_version, volumes, image_ids, true)
-            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Get"))?;
-        self.set
-            .validate(eos_version, volumes, image_ids, true)
-            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Set"))?;
-        Ok(())
-    }
-    #[instrument(skip_all)]
-    pub async fn get(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        volumes: &Volumes,
-    ) -> Result<ConfigRes, Error> {
-        self.get
-            .execute(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::GetConfig,
-                volumes,
-                None::<()>,
-                None,
-            )
-            .await
-            .and_then(|res| {
-                res.map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::ConfigGen))
-            })
-    }
-
-    #[instrument(skip_all)]
-    pub async fn set(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        dependencies: &Dependencies,
-        volumes: &Volumes,
-        input: &Config,
-    ) -> Result<SetResult, Error> {
-        let res: SetResult = self
-            .set
-            .execute(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::SetConfig,
-                volumes,
-                Some(input),
-                None,
-            )
-            .await
-            .and_then(|res| {
-                res.map_err(|e| {
-                    Error::new(eyre!("{}", e.1), crate::ErrorKind::ConfigRulesViolation)
-                })
-            })?;
-        Ok(SetResult {
-            depends_on: res
-                .depends_on
-                .into_iter()
-                .filter(|(pkg, _)| dependencies.0.contains_key(pkg))
-                .collect(),
-        })
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct SetResult {
-    pub depends_on: BTreeMap<PackageId, BTreeSet<HealthCheckId>>,
-}

backend/src/config/mod.rs

@@ -1,287 +0,0 @@
-use std::collections::BTreeMap;
-use std::path::PathBuf;
-use std::sync::Arc;
-use std::time::Duration;
-
-use color_eyre::eyre::eyre;
-use indexmap::IndexSet;
-use itertools::Itertools;
-use models::{ErrorKind, OptionExt};
-use patch_db::value::InternedString;
-use patch_db::Value;
-use regex::Regex;
-use rpc_toolkit::command;
-use tracing::instrument;
-
-use crate::context::RpcContext;
-use crate::prelude::*;
-use crate::s9pk::manifest::PackageId;
-use crate::util::display_none;
-use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
-use crate::Error;
-
-pub mod action;
-pub mod spec;
-pub mod util;
-
-pub use spec::{ConfigSpec, Defaultable};
-use util::NumRange;
-
-use self::action::ConfigRes;
-use self::spec::ValueSpecPointer;
-
-pub type Config = patch_db::value::InOMap<InternedString, Value>;
-pub trait TypeOf {
-    fn type_of(&self) -> &'static str;
-}
-impl TypeOf for Value {
-    fn type_of(&self) -> &'static str {
-        match self {
-            Value::Array(_) => "list",
-            Value::Bool(_) => "boolean",
-            Value::Null => "null",
-            Value::Number(_) => "number",
-            Value::Object(_) => "object",
-            Value::String(_) => "string",
-        }
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum ConfigurationError {
-    #[error("Timeout Error")]
-    TimeoutError(#[from] TimeoutError),
-    #[error("No Match: {0}")]
-    NoMatch(#[from] NoMatchWithPath),
-    #[error("System Error: {0}")]
-    SystemError(Error),
-    #[error("Permission Denied: {0}")]
-    PermissionDenied(ValueSpecPointer),
-}
-impl From<ConfigurationError> for Error {
-    fn from(err: ConfigurationError) -> Self {
-        let kind = match &err {
-            ConfigurationError::SystemError(e) => e.kind,
-            _ => crate::ErrorKind::ConfigGen,
-        };
-        crate::Error::new(err, kind)
-    }
-}
-
-#[derive(Clone, Copy, Debug, thiserror::Error)]
-#[error("Timeout Error")]
-pub struct TimeoutError;
-
-#[derive(Clone, Debug, thiserror::Error)]
-pub struct NoMatchWithPath {
-    pub path: Vec<InternedString>,
-    pub error: MatchError,
-}
-impl NoMatchWithPath {
-    pub fn new(error: MatchError) -> Self {
-        NoMatchWithPath {
-            path: Vec::new(),
-            error,
-        }
-    }
-    pub fn prepend(mut self, seg: InternedString) -> Self {
-        self.path.push(seg);
-        self
-    }
-}
-impl std::fmt::Display for NoMatchWithPath {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}: {}", self.path.iter().rev().join("."), self.error)
-    }
-}
-impl From<NoMatchWithPath> for Error {
-    fn from(e: NoMatchWithPath) -> Self {
-        ConfigurationError::from(e).into()
-    }
-}
-
-#[derive(Clone, Debug, thiserror::Error)]
-pub enum MatchError {
-    #[error("String {0:?} Does Not Match Pattern {1}")]
-    Pattern(Arc<String>, Regex),
-    #[error("String {0:?} Is Not In Enum {1:?}")]
-    Enum(Arc<String>, IndexSet<String>),
-    #[error("Field Is Not Nullable")]
-    NotNullable,
-    #[error("Length Mismatch: expected {0}, actual: {1}")]
-    LengthMismatch(NumRange<usize>, usize),
-    #[error("Invalid Type: expected {0}, actual: {1}")]
-    InvalidType(&'static str, &'static str),
-    #[error("Number Out Of Range: expected {0}, actual: {1}")]
-    OutOfRange(NumRange<f64>, f64),
-    #[error("Number Is Not Integral: {0}")]
-    NonIntegral(f64),
-    #[error("Variant {0:?} Is Not In Union {1:?}")]
-    Union(Arc<String>, IndexSet<String>),
-    #[error("Variant Is Missing Tag {0:?}")]
-    MissingTag(InternedString),
-    #[error("Property {0:?} Of Variant {1:?} Conflicts With Union Tag")]
-    PropertyMatchesUnionTag(InternedString, String),
-    #[error("Name of Property {0:?} Conflicts With Map Tag Name")]
-    PropertyNameMatchesMapTag(String),
-    #[error("Pointer Is Invalid: {0}")]
-    InvalidPointer(spec::ValueSpecPointer),
-    #[error("Object Key Is Invalid: {0}")]
-    InvalidKey(String),
-    #[error("Value In List Is Not Unique")]
-    ListUniquenessViolation,
-}
-
-#[command(rename = "config-spec", cli_only, blocking, display(display_none))]
-pub fn verify_spec(#[arg] path: PathBuf) -> Result<(), Error> {
-    let mut file = std::fs::File::open(&path)?;
-    let format = match path.extension().and_then(|s| s.to_str()) {
-        Some("yaml") | Some("yml") => IoFormat::Yaml,
-        Some("json") => IoFormat::Json,
-        Some("toml") => IoFormat::Toml,
-        Some("cbor") => IoFormat::Cbor,
-        _ => {
-            return Err(Error::new(
-                eyre!("Unknown file format. Expected one of yaml, json, toml, cbor."),
-                crate::ErrorKind::Deserialization,
-            ));
-        }
-    };
-    let _: ConfigSpec = format.from_reader(&mut file)?;
-
-    Ok(())
-}
-
-#[command(subcommands(get, set))]
-pub fn config(#[arg] id: PackageId) -> Result<PackageId, Error> {
-    Ok(id)
-}
-
-#[command(display(display_serializable))]
-#[instrument(skip_all)]
-pub async fn get(
-    #[context] ctx: RpcContext,
-    #[parent_data] id: PackageId,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-) -> Result<ConfigRes, Error> {
-    let db = ctx.db.peek().await;
-    let manifest = db
-        .as_package_data()
-        .as_idx(&id)
-        .or_not_found(&id)?
-        .as_installed()
-        .or_not_found(&id)?
-        .as_manifest();
-    let action = manifest
-        .as_config()
-        .de()?
-        .ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
-
-    let volumes = manifest.as_volumes().de()?;
-    let version = manifest.as_version().de()?;
-    action.get(&ctx, &id, &version, &volumes).await
-}
-
-#[command(
-    subcommands(self(set_impl(async, context(RpcContext))), set_dry),
-    display(display_none),
-    metadata(sync_db = true)
-)]
-#[instrument(skip_all)]
-pub fn set(
-    #[parent_data] id: PackageId,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-    #[arg(long = "timeout")] timeout: Option<crate::util::serde::Duration>,
-    #[arg(stdin, parse(parse_stdin_deserializable))] config: Option<Config>,
-) -> Result<(PackageId, Option<Config>, Option<Duration>), Error> {
-    Ok((id, config, timeout.map(|d| *d)))
-}
-
-#[command(rename = "dry", display(display_serializable))]
-#[instrument(skip_all)]
-pub async fn set_dry(
-    #[context] ctx: RpcContext,
-    #[parent_data] (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
-) -> Result<BTreeMap<PackageId, String>, Error> {
-    let breakages = BTreeMap::new();
-    let overrides = Default::default();
-
-    let configure_context = ConfigureContext {
-        breakages,
-        timeout,
-        config,
-        dry_run: true,
-        overrides,
-    };
-    let breakages = configure(&ctx, &id, configure_context).await?;
-
-    Ok(breakages)
-}
-
-pub struct ConfigureContext {
-    pub breakages: BTreeMap<PackageId, String>,
-    pub timeout: Option<Duration>,
-    pub config: Option<Config>,
-    pub overrides: BTreeMap<PackageId, Config>,
-    pub dry_run: bool,
-}
-
-#[instrument(skip_all)]
-pub async fn set_impl(
-    ctx: RpcContext,
-    (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
-) -> Result<(), Error> {
-    let breakages = BTreeMap::new();
-    let overrides = Default::default();
-
-    let configure_context = ConfigureContext {
-        breakages,
-        timeout,
-        config,
-        dry_run: false,
-        overrides,
-    };
-    configure(&ctx, &id, configure_context).await?;
-    Ok(())
-}
-
-#[instrument(skip_all)]
-pub async fn configure(
-    ctx: &RpcContext,
-    id: &PackageId,
-    configure_context: ConfigureContext,
-) -> Result<BTreeMap<PackageId, String>, Error> {
-    let db = ctx.db.peek().await;
-    let package = db
-        .as_package_data()
-        .as_idx(id)
-        .or_not_found(&id)?
-        .as_installed()
-        .or_not_found(&id)?;
-    let version = package.as_manifest().as_version().de()?;
-    ctx.managers
-        .get(&(id.clone(), version.clone()))
-        .await
-        .ok_or_else(|| {
-            Error::new(
-                eyre!("There is no manager running for {id:?} and {version:?}"),
-                ErrorKind::Unknown,
-            )
-        })?
-        .configure(configure_context)
-        .await
-}
-
-macro_rules! not_found {
-    ($x:expr) => {
-        crate::Error::new(
-            color_eyre::eyre::eyre!("Could not find {} at {}:{}", $x, module_path!(), line!()),
-            crate::ErrorKind::Incoherent,
-        )
-    };
-}
-pub(crate) use not_found;

backend/src/config/util.rs

@@ -1,406 +0,0 @@
-use std::borrow::Cow;
-use std::ops::{Bound, RangeBounds, RangeInclusive};
-
-use patch_db::Value;
-use rand::distributions::Distribution;
-use rand::Rng;
-
-use super::Config;
-
-pub const STATIC_NULL: Value = Value::Null;
-
-#[derive(Clone, Debug)]
-pub struct CharSet(pub Vec<(RangeInclusive<char>, usize)>, usize);
-impl CharSet {
-    pub fn contains(&self, c: &char) -> bool {
-        self.0.iter().any(|r| r.0.contains(c))
-    }
-    pub fn gen<R: Rng>(&self, rng: &mut R) -> char {
-        let mut idx = rng.gen_range(0..self.1);
-        for r in &self.0 {
-            if idx < r.1 {
-                return std::convert::TryFrom::try_from(
-                    rand::distributions::Uniform::new_inclusive(
-                        u32::from(*r.0.start()),
-                        u32::from(*r.0.end()),
-                    )
-                    .sample(rng),
-                )
-                .unwrap();
-            } else {
-                idx -= r.1;
-            }
-        }
-        unreachable!()
-    }
-}
-impl Default for CharSet {
-    fn default() -> Self {
-        CharSet(vec![('!'..='~', 94)], 94)
-    }
-}
-impl<'de> serde::de::Deserialize<'de> for CharSet {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::de::Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        let mut res = Vec::new();
-        let mut len = 0;
-        let mut a: Option<char> = None;
-        let mut b: Option<char> = None;
-        let mut in_range = false;
-        for c in s.chars() {
-            match c {
-                ',' => match (a, b, in_range) {
-                    (Some(start), Some(end), _) => {
-                        if !end.is_ascii() {
-                            return Err(serde::de::Error::custom("Invalid Character"));
-                        }
-                        if start >= end {
-                            return Err(serde::de::Error::custom("Invalid Bounds"));
-                        }
-                        let l = u32::from(end) - u32::from(start) + 1;
-                        res.push((start..=end, l as usize));
-                        len += l as usize;
-                        a = None;
-                        b = None;
-                        in_range = false;
-                    }
-                    (Some(start), None, false) => {
-                        len += 1;
-                        res.push((start..=start, 1));
-                        a = None;
-                    }
-                    (Some(_), None, true) => {
-                        b = Some(',');
-                    }
-                    (None, None, false) => {
-                        a = Some(',');
-                    }
-                    _ => {
-                        return Err(serde::de::Error::custom("Syntax Error"));
-                    }
-                },
-                '-' => {
-                    if a.is_none() {
-                        a = Some('-');
-                    } else if !in_range {
-                        in_range = true;
-                    } else if b.is_none() {
-                        b = Some('-')
-                    } else {
-                        return Err(serde::de::Error::custom("Syntax Error"));
-                    }
-                }
-                _ => {
-                    if a.is_none() {
-                        a = Some(c);
-                    } else if in_range && b.is_none() {
-                        b = Some(c);
-                    } else {
-                        return Err(serde::de::Error::custom("Syntax Error"));
-                    }
-                }
-            }
-        }
-        match (a, b) {
-            (Some(start), Some(end)) => {
-                if !end.is_ascii() {
-                    return Err(serde::de::Error::custom("Invalid Character"));
-                }
-                if start >= end {
-                    return Err(serde::de::Error::custom("Invalid Bounds"));
-                }
-                let l = u32::from(end) - u32::from(start) + 1;
-                res.push((start..=end, l as usize));
-                len += l as usize;
-            }
-            (Some(c), None) => {
-                len += 1;
-                res.push((c..=c, 1));
-            }
-            _ => (),
-        }
-
-        Ok(CharSet(res, len))
-    }
-}
-impl serde::ser::Serialize for CharSet {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::ser::Serializer,
-    {
-        <&str>::serialize(
-            &self
-                .0
-                .iter()
-                .map(|r| match r.1 {
-                    1 => format!("{}", r.0.start()),
-                    _ => format!("{}-{}", r.0.start(), r.0.end()),
-                })
-                .collect::<Vec<_>>()
-                .join(",")
-                .as_str(),
-            serializer,
-        )
-    }
-}
-
-pub trait MergeWith {
-    fn merge_with(&mut self, other: &serde_json::Value);
-}
-
-impl MergeWith for serde_json::Value {
-    fn merge_with(&mut self, other: &serde_json::Value) {
-        use serde_json::Value::Object;
-        if let (Object(orig), Object(ref other)) = (self, other) {
-            for (key, val) in other.into_iter() {
-                match (orig.get_mut(key), val) {
-                    (Some(new_orig @ Object(_)), other @ Object(_)) => {
-                        new_orig.merge_with(other);
-                    }
-                    (None, _) => {
-                        orig.insert(key.clone(), val.clone());
-                    }
-                    _ => (),
-                }
-            }
-        }
-    }
-}
-
-#[test]
-fn merge_with_tests() {
-    use serde_json::json;
-
-    let mut a = json!(
-        {"a": 1, "c": {"d": "123"}, "i": [1,2,3], "j": {}, "k":[1,2,3], "l": "test"}
-    );
-    a.merge_with(
-        &json!({"a":"a", "b": "b", "c":{"d":"d", "e":"e"}, "f":{"g":"g"}, "h": [1,2,3], "i":"i", "j":[1,2,3], "k":{}}),
-    );
-    assert_eq!(
-        a,
-        json!({"a": 1, "c": {"d": "123", "e":"e"}, "b":"b", "f": {"g":"g"}, "h":[1,2,3], "i":[1,2,3], "j": {}, "k":[1,2,3], "l": "test"})
-    )
-}
-pub mod serde_regex {
-    use regex::Regex;
-    use serde::*;
-
-    pub fn serialize<S>(regex: &Regex, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        <&str>::serialize(&regex.as_str(), serializer)
-    }
-
-    pub fn deserialize<'de, D>(deserializer: D) -> Result<Regex, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        Regex::new(&s).map_err(|e| de::Error::custom(e))
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct NumRange<T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd>(
-    pub (Bound<T>, Bound<T>),
-);
-impl<T> std::ops::Deref for NumRange<T>
-where
-    T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd,
-{
-    type Target = (Bound<T>, Bound<T>);
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-impl<'de, T> serde::de::Deserialize<'de> for NumRange<T>
-where
-    T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd,
-    <T as std::str::FromStr>::Err: std::fmt::Display,
-{
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::de::Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        let mut split = s.split(",");
-        let start = split
-            .next()
-            .map(|s| match s.get(..1) {
-                Some("(") => match s.get(1..2) {
-                    Some("*") => Ok(Bound::Unbounded),
-                    _ => s[1..]
-                        .trim()
-                        .parse()
-                        .map(Bound::Excluded)
-                        .map_err(|e| serde::de::Error::custom(e)),
-                },
-                Some("[") => s[1..]
-                    .trim()
-                    .parse()
-                    .map(Bound::Included)
-                    .map_err(|e| serde::de::Error::custom(e)),
-                _ => Err(serde::de::Error::custom(format!(
-                    "Could not parse left bound: {}",
-                    s
-                ))),
-            })
-            .transpose()?
-            .unwrap();
-        let end = split
-            .next()
-            .map(|s| match s.get(s.len() - 1..) {
-                Some(")") => match s.get(s.len() - 2..s.len() - 1) {
-                    Some("*") => Ok(Bound::Unbounded),
-                    _ => s[..s.len() - 1]
-                        .trim()
-                        .parse()
-                        .map(Bound::Excluded)
-                        .map_err(|e| serde::de::Error::custom(e)),
-                },
-                Some("]") => s[..s.len() - 1]
-                    .trim()
-                    .parse()
-                    .map(Bound::Included)
-                    .map_err(|e| serde::de::Error::custom(e)),
-                _ => Err(serde::de::Error::custom(format!(
-                    "Could not parse right bound: {}",
-                    s
-                ))),
-            })
-            .transpose()?
-            .unwrap_or(Bound::Unbounded);
-
-        Ok(NumRange((start, end)))
-    }
-}
-impl<T> std::fmt::Display for NumRange<T>
-where
-    T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self.start_bound() {
-            Bound::Excluded(n) => write!(f, "({},", n)?,
-            Bound::Included(n) => write!(f, "[{},", n)?,
-            Bound::Unbounded => write!(f, "(*,")?,
-        };
-        match self.end_bound() {
-            Bound::Excluded(n) => write!(f, "{})", n),
-            Bound::Included(n) => write!(f, "{}]", n),
-            Bound::Unbounded => write!(f, "*)"),
-        }
-    }
-}
-impl<T> serde::ser::Serialize for NumRange<T>
-where
-    T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd,
-{
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::ser::Serializer,
-    {
-        <&str>::serialize(&format!("{}", self).as_str(), serializer)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub enum UniqueBy {
-    Any(Vec<UniqueBy>),
-    All(Vec<UniqueBy>),
-    Exactly(String),
-    NotUnique,
-}
-impl UniqueBy {
-    pub fn eq(&self, lhs: &Config, rhs: &Config) -> bool {
-        match self {
-            UniqueBy::Any(any) => any.iter().any(|u| u.eq(lhs, rhs)),
-            UniqueBy::All(all) => all.iter().all(|u| u.eq(lhs, rhs)),
-            UniqueBy::Exactly(key) => lhs.get(&**key) == rhs.get(&**key),
-            UniqueBy::NotUnique => false,
-        }
-    }
-}
-impl Default for UniqueBy {
-    fn default() -> Self {
-        UniqueBy::NotUnique
-    }
-}
-impl<'de> serde::de::Deserialize<'de> for UniqueBy {
-    fn deserialize<D: serde::de::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
-        struct Visitor;
-        impl<'de> serde::de::Visitor<'de> for Visitor {
-            type Value = UniqueBy;
-            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-                write!(formatter, "a key, an \"any\" object, or an \"all\" object")
-            }
-            fn visit_str<E: serde::de::Error>(self, v: &str) -> Result<Self::Value, E> {
-                Ok(UniqueBy::Exactly(v.to_owned()))
-            }
-            fn visit_string<E: serde::de::Error>(self, v: String) -> Result<Self::Value, E> {
-                Ok(UniqueBy::Exactly(v))
-            }
-            fn visit_map<A: serde::de::MapAccess<'de>>(
-                self,
-                mut map: A,
-            ) -> Result<Self::Value, A::Error> {
-                let mut variant = None;
-                while let Some(key) = map.next_key::<Cow<str>>()? {
-                    match key.as_ref() {
-                        "any" => {
-                            return Ok(UniqueBy::Any(map.next_value()?));
-                        }
-                        "all" => {
-                            return Ok(UniqueBy::All(map.next_value()?));
-                        }
-                        _ => {
-                            variant = Some(key);
-                        }
-                    }
-                }
-                Err(serde::de::Error::unknown_variant(
-                    variant.unwrap_or_default().as_ref(),
-                    &["any", "all"],
-                ))
-            }
-            fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> {
-                Ok(UniqueBy::NotUnique)
-            }
-            fn visit_none<E: serde::de::Error>(self) -> Result<Self::Value, E> {
-                Ok(UniqueBy::NotUnique)
-            }
-        }
-        deserializer.deserialize_any(Visitor)
-    }
-}
-
-impl serde::ser::Serialize for UniqueBy {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::ser::Serializer,
-    {
-        use serde::ser::SerializeMap;
-
-        match self {
-            UniqueBy::Any(any) => {
-                let mut map = serializer.serialize_map(Some(1))?;
-                map.serialize_key("any")?;
-                map.serialize_value(any)?;
-                map.end()
-            }
-            UniqueBy::All(all) => {
-                let mut map = serializer.serialize_map(Some(1))?;
-                map.serialize_key("all")?;
-                map.serialize_value(all)?;
-                map.end()
-            }
-            UniqueBy::Exactly(key) => serializer.serialize_str(key),
-            UniqueBy::NotUnique => serializer.serialize_unit(),
-        }
-    }
-}

backend/src/context/cli.rs

@@ -1,185 +0,0 @@
-use std::fs::File;
-use std::io::BufReader;
-use std::net::Ipv4Addr;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use clap::ArgMatches;
-use color_eyre::eyre::eyre;
-use cookie_store::{CookieStore, RawCookie};
-use josekit::jwk::Jwk;
-use reqwest::Proxy;
-use reqwest_cookie_store::CookieStoreMutex;
-use rpc_toolkit::reqwest::{Client, Url};
-use rpc_toolkit::url::Host;
-use rpc_toolkit::Context;
-use serde::Deserialize;
-use tracing::instrument;
-
-use super::setup::CURRENT_SECRET;
-use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
-use crate::util::config::{load_config_from_paths, local_config_path};
-use crate::ResultExt;
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct CliContextConfig {
-    pub host: Option<Url>,
-    #[serde(deserialize_with = "crate::util::serde::deserialize_from_str_opt")]
-    #[serde(default)]
-    pub proxy: Option<Url>,
-    pub cookie_path: Option<PathBuf>,
-}
-
-#[derive(Debug)]
-pub struct CliContextSeed {
-    pub base_url: Url,
-    pub rpc_url: Url,
-    pub client: Client,
-    pub cookie_store: Arc<CookieStoreMutex>,
-    pub cookie_path: PathBuf,
-}
-impl Drop for CliContextSeed {
-    fn drop(&mut self) {
-        let tmp = format!("{}.tmp", self.cookie_path.display());
-        let parent_dir = self.cookie_path.parent().unwrap_or(Path::new("/"));
-        if !parent_dir.exists() {
-            std::fs::create_dir_all(&parent_dir).unwrap();
-        }
-        let mut writer = fd_lock_rs::FdLock::lock(
-            File::create(&tmp).unwrap(),
-            fd_lock_rs::LockType::Exclusive,
-            true,
-        )
-        .unwrap();
-        let mut store = self.cookie_store.lock().unwrap();
-        store.remove("localhost", "", "local");
-        store.save_json(&mut *writer).unwrap();
-        writer.sync_all().unwrap();
-        std::fs::rename(tmp, &self.cookie_path).unwrap();
-    }
-}
-
-const DEFAULT_HOST: Host<&'static str> = Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1));
-const DEFAULT_PORT: u16 = 5959;
-
-#[derive(Debug, Clone)]
-pub struct CliContext(Arc<CliContextSeed>);
-impl CliContext {
-    /// BLOCKING
-    #[instrument(skip_all)]
-    pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
-        let local_config_path = local_config_path();
-        let base: CliContextConfig = load_config_from_paths(
-            matches
-                .values_of("config")
-                .into_iter()
-                .flatten()
-                .map(|p| Path::new(p))
-                .chain(local_config_path.as_deref().into_iter())
-                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
-        )?;
-        let mut url = if let Some(host) = matches.value_of("host") {
-            host.parse()?
-        } else if let Some(host) = base.host {
-            host
-        } else {
-            "http://localhost".parse()?
-        };
-        let proxy = if let Some(proxy) = matches.value_of("proxy") {
-            Some(proxy.parse()?)
-        } else {
-            base.proxy
-        };
-
-        let cookie_path = base.cookie_path.unwrap_or_else(|| {
-            local_config_path
-                .as_deref()
-                .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
-                .parent()
-                .unwrap_or(Path::new("/"))
-                .join(".cookies.json")
-        });
-        let cookie_store = Arc::new(CookieStoreMutex::new({
-            let mut store = if cookie_path.exists() {
-                CookieStore::load_json(BufReader::new(File::open(&cookie_path)?))
-                    .map_err(|e| eyre!("{}", e))
-                    .with_kind(crate::ErrorKind::Deserialization)?
-            } else {
-                CookieStore::default()
-            };
-            if let Ok(local) = std::fs::read_to_string(LOCAL_AUTH_COOKIE_PATH) {
-                store
-                    .insert_raw(
-                        &RawCookie::new("local", local),
-                        &"http://localhost".parse()?,
-                    )
-                    .with_kind(crate::ErrorKind::Network)?;
-            }
-            store
-        }));
-
-        Ok(CliContext(Arc::new(CliContextSeed {
-            base_url: url.clone(),
-            rpc_url: {
-                url.path_segments_mut()
-                    .map_err(|_| eyre!("Url cannot be base"))
-                    .with_kind(crate::ErrorKind::ParseUrl)?
-                    .push("rpc")
-                    .push("v1");
-                url
-            },
-            client: {
-                let mut builder = Client::builder().cookie_provider(cookie_store.clone());
-                if let Some(proxy) = proxy {
-                    builder =
-                        builder.proxy(Proxy::all(proxy).with_kind(crate::ErrorKind::ParseUrl)?)
-                }
-                builder.build().expect("cannot fail")
-            },
-            cookie_store,
-            cookie_path,
-        })))
-    }
-}
-impl AsRef<Jwk> for CliContext {
-    fn as_ref(&self) -> &Jwk {
-        &*CURRENT_SECRET
-    }
-}
-impl std::ops::Deref for CliContext {
-    type Target = CliContextSeed;
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}
-impl Context for CliContext {
-    fn protocol(&self) -> &str {
-        self.0.base_url.scheme()
-    }
-    fn host(&self) -> Host<&str> {
-        self.0.base_url.host().unwrap_or(DEFAULT_HOST)
-    }
-    fn port(&self) -> u16 {
-        self.0.base_url.port().unwrap_or(DEFAULT_PORT)
-    }
-    fn path(&self) -> &str {
-        self.0.rpc_url.path()
-    }
-    fn url(&self) -> Url {
-        self.0.rpc_url.clone()
-    }
-    fn client(&self) -> &Client {
-        &self.0.client
-    }
-}
-/// When we had an empty proxy the system wasn't working like it used to, which allowed empty proxy
-#[test]
-fn test_cli_proxy_empty() {
-    serde_yaml::from_str::<CliContextConfig>(
-        "
-        bind_rpc:
-    ",
-    )
-    .unwrap();
-}

backend/src/context/diagnostic.rs

@@ -1,83 +0,0 @@
-use std::ops::Deref;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::Context;
-use serde::Deserialize;
-use tokio::sync::broadcast::Sender;
-use tracing::instrument;
-
-use crate::shutdown::Shutdown;
-use crate::util::config::load_config_from_paths;
-use crate::Error;
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct DiagnosticContextConfig {
-    pub datadir: Option<PathBuf>,
-}
-impl DiagnosticContextConfig {
-    #[instrument(skip_all)]
-    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
-        tokio::task::spawn_blocking(move || {
-            load_config_from_paths(
-                path.as_ref()
-                    .into_iter()
-                    .map(|p| p.as_ref())
-                    .chain(std::iter::once(Path::new(
-                        crate::util::config::DEVICE_CONFIG_PATH,
-                    )))
-                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
-            )
-        })
-        .await
-        .unwrap()
-    }
-
-    pub fn datadir(&self) -> &Path {
-        self.datadir
-            .as_deref()
-            .unwrap_or_else(|| Path::new("/embassy-data"))
-    }
-}
-
-pub struct DiagnosticContextSeed {
-    pub datadir: PathBuf,
-    pub shutdown: Sender<Option<Shutdown>>,
-    pub error: Arc<RpcError>,
-    pub disk_guid: Option<Arc<String>>,
-}
-
-#[derive(Clone)]
-pub struct DiagnosticContext(Arc<DiagnosticContextSeed>);
-impl DiagnosticContext {
-    #[instrument(skip_all)]
-    pub async fn init<P: AsRef<Path> + Send + 'static>(
-        path: Option<P>,
-        disk_guid: Option<Arc<String>>,
-        error: Error,
-    ) -> Result<Self, Error> {
-        tracing::error!("Error: {}: Starting diagnostic UI", error);
-        tracing::debug!("{:?}", error);
-
-        let cfg = DiagnosticContextConfig::load(path).await?;
-
-        let (shutdown, _) = tokio::sync::broadcast::channel(1);
-
-        Ok(Self(Arc::new(DiagnosticContextSeed {
-            datadir: cfg.datadir().to_owned(),
-            shutdown,
-            disk_guid,
-            error: Arc::new(error.into()),
-        })))
-    }
-}
-
-impl Context for DiagnosticContext {}
-impl Deref for DiagnosticContext {
-    type Target = DiagnosticContextSeed;
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}

backend/src/context/install.rs

@@ -1,58 +0,0 @@
-use std::ops::Deref;
-use std::path::Path;
-use std::sync::Arc;
-
-use rpc_toolkit::Context;
-use serde::Deserialize;
-use tokio::sync::broadcast::Sender;
-use tracing::instrument;
-
-use crate::net::utils::find_eth_iface;
-use crate::util::config::load_config_from_paths;
-use crate::Error;
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct InstallContextConfig {}
-impl InstallContextConfig {
-    #[instrument(skip_all)]
-    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
-        tokio::task::spawn_blocking(move || {
-            load_config_from_paths(
-                path.as_ref()
-                    .into_iter()
-                    .map(|p| p.as_ref())
-                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
-            )
-        })
-        .await
-        .unwrap()
-    }
-}
-
-pub struct InstallContextSeed {
-    pub ethernet_interface: String,
-    pub shutdown: Sender<()>,
-}
-
-#[derive(Clone)]
-pub struct InstallContext(Arc<InstallContextSeed>);
-impl InstallContext {
-    #[instrument(skip_all)]
-    pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
-        let _cfg = InstallContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
-        let (shutdown, _) = tokio::sync::broadcast::channel(1);
-        Ok(Self(Arc::new(InstallContextSeed {
-            ethernet_interface: find_eth_iface().await?,
-            shutdown,
-        })))
-    }
-}
-
-impl Context for InstallContext {}
-impl Deref for InstallContext {
-    type Target = InstallContextSeed;
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}

backend/src/context/mod.rs

@@ -1,44 +0,0 @@
-pub mod cli;
-pub mod diagnostic;
-pub mod install;
-pub mod rpc;
-pub mod sdk;
-pub mod setup;
-
-pub use cli::CliContext;
-pub use diagnostic::DiagnosticContext;
-pub use install::InstallContext;
-pub use rpc::RpcContext;
-pub use sdk::SdkContext;
-pub use setup::SetupContext;
-
-impl From<CliContext> for () {
-    fn from(_: CliContext) -> Self {
-        ()
-    }
-}
-impl From<DiagnosticContext> for () {
-    fn from(_: DiagnosticContext) -> Self {
-        ()
-    }
-}
-impl From<RpcContext> for () {
-    fn from(_: RpcContext) -> Self {
-        ()
-    }
-}
-impl From<SdkContext> for () {
-    fn from(_: SdkContext) -> Self {
-        ()
-    }
-}
-impl From<SetupContext> for () {
-    fn from(_: SetupContext) -> Self {
-        ()
-    }
-}
-impl From<InstallContext> for () {
-    fn from(_: InstallContext) -> Self {
-        ()
-    }
-}

backend/src/context/rpc.rs

@@ -1,466 +0,0 @@
-use std::collections::BTreeMap;
-use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
-use std::ops::Deref;
-use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-
-use helpers::to_tmp_path;
-use josekit::jwk::Jwk;
-use patch_db::json_ptr::JsonPointer;
-use patch_db::PatchDb;
-use reqwest::{Client, Proxy, Url};
-use rpc_toolkit::Context;
-use serde::Deserialize;
-use sqlx::postgres::PgConnectOptions;
-use sqlx::PgPool;
-use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
-use tokio::time::Instant;
-use tracing::instrument;
-
-use super::setup::CURRENT_SECRET;
-use crate::account::AccountInfo;
-use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation};
-use crate::db::model::{CurrentDependents, Database, PackageDataEntryMatchModelRef};
-use crate::db::prelude::PatchDbExt;
-use crate::dependencies::compute_dependency_config_errs;
-use crate::disk::OsPartitionInfo;
-use crate::init::init_postgres;
-use crate::install::cleanup::{cleanup_failed, uninstall};
-use crate::manager::ManagerMap;
-use crate::middleware::auth::HashSessionToken;
-use crate::net::net_controller::NetController;
-use crate::net::ssl::{root_ca_start_time, SslManager};
-use crate::net::wifi::WpaCli;
-use crate::notifications::NotificationManager;
-use crate::shutdown::Shutdown;
-use crate::status::MainStatus;
-use crate::system::get_mem_info;
-use crate::util::config::load_config_from_paths;
-use crate::util::lshw::{lshw, LshwDevice};
-use crate::{Error, ErrorKind, ResultExt};
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct RpcContextConfig {
-    pub wifi_interface: Option<String>,
-    pub ethernet_interface: String,
-    pub os_partitions: OsPartitionInfo,
-    pub migration_batch_rows: Option<usize>,
-    pub migration_prefetch_rows: Option<usize>,
-    pub bind_rpc: Option<SocketAddr>,
-    pub tor_control: Option<SocketAddr>,
-    pub tor_socks: Option<SocketAddr>,
-    pub dns_bind: Option<Vec<SocketAddr>>,
-    pub revision_cache_size: Option<usize>,
-    pub datadir: Option<PathBuf>,
-    pub log_server: Option<Url>,
-}
-impl RpcContextConfig {
-    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
-        tokio::task::spawn_blocking(move || {
-            load_config_from_paths(
-                path.as_ref()
-                    .into_iter()
-                    .map(|p| p.as_ref())
-                    .chain(std::iter::once(Path::new(
-                        crate::util::config::DEVICE_CONFIG_PATH,
-                    )))
-                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
-            )
-        })
-        .await
-        .unwrap()
-    }
-    pub fn datadir(&self) -> &Path {
-        self.datadir
-            .as_deref()
-            .unwrap_or_else(|| Path::new("/embassy-data"))
-    }
-    pub async fn db(&self, account: &AccountInfo) -> Result<PatchDb, Error> {
-        let db_path = self.datadir().join("main").join("embassy.db");
-        let db = PatchDb::open(&db_path)
-            .await
-            .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
-        if !db.exists(&<JsonPointer>::default()).await {
-            db.put(&<JsonPointer>::default(), &Database::init(account))
-                .await?;
-        }
-        Ok(db)
-    }
-    #[instrument(skip_all)]
-    pub async fn secret_store(&self) -> Result<PgPool, Error> {
-        init_postgres(self.datadir()).await?;
-        let secret_store =
-            PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
-                .await?;
-        sqlx::migrate!()
-            .run(&secret_store)
-            .await
-            .with_kind(crate::ErrorKind::Database)?;
-        Ok(secret_store)
-    }
-}
-
-pub struct RpcContextSeed {
-    is_closed: AtomicBool,
-    pub os_partitions: OsPartitionInfo,
-    pub wifi_interface: Option<String>,
-    pub ethernet_interface: String,
-    pub datadir: PathBuf,
-    pub disk_guid: Arc<String>,
-    pub db: PatchDb,
-    pub secret_store: PgPool,
-    pub account: RwLock<AccountInfo>,
-    pub net_controller: Arc<NetController>,
-    pub managers: ManagerMap,
-    pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
-    pub shutdown: broadcast::Sender<Option<Shutdown>>,
-    pub tor_socks: SocketAddr,
-    pub notification_manager: NotificationManager,
-    pub open_authed_websockets: Mutex<BTreeMap<HashSessionToken, Vec<oneshot::Sender<()>>>>,
-    pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
-    pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
-    pub current_secret: Arc<Jwk>,
-    pub client: Client,
-    pub hardware: Hardware,
-    pub start_time: Instant,
-}
-
-pub struct Hardware {
-    pub devices: Vec<LshwDevice>,
-    pub ram: u64,
-}
-
-#[derive(Clone)]
-pub struct RpcContext(Arc<RpcContextSeed>);
-impl RpcContext {
-    #[instrument(skip_all)]
-    pub async fn init<P: AsRef<Path> + Send + Sync + 'static>(
-        cfg_path: Option<P>,
-        disk_guid: Arc<String>,
-    ) -> Result<Self, Error> {
-        let base = RpcContextConfig::load(cfg_path).await?;
-        tracing::info!("Loaded Config");
-        let tor_proxy = base.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new(
-            Ipv4Addr::new(127, 0, 0, 1),
-            9050,
-        )));
-        let (shutdown, _) = tokio::sync::broadcast::channel(1);
-        let secret_store = base.secret_store().await?;
-        tracing::info!("Opened Pg DB");
-        let account = AccountInfo::load(&secret_store).await?;
-        let db = base.db(&account).await?;
-        tracing::info!("Opened PatchDB");
-        let net_controller = Arc::new(
-            NetController::init(
-                base.tor_control
-                    .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
-                tor_proxy,
-                base.dns_bind
-                    .as_deref()
-                    .unwrap_or(&[SocketAddr::from(([127, 0, 0, 1], 53))]),
-                SslManager::new(&account, root_ca_start_time().await?)?,
-                &account.hostname,
-                &account.key,
-            )
-            .await?,
-        );
-        tracing::info!("Initialized Net Controller");
-        let managers = ManagerMap::default();
-        let metrics_cache = RwLock::<Option<crate::system::Metrics>>::new(None);
-        let notification_manager = NotificationManager::new(secret_store.clone());
-        tracing::info!("Initialized Notification Manager");
-        let tor_proxy_url = format!("socks5h://{tor_proxy}");
-        let devices = lshw().await?;
-        let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
-        let seed = Arc::new(RpcContextSeed {
-            is_closed: AtomicBool::new(false),
-            datadir: base.datadir().to_path_buf(),
-            os_partitions: base.os_partitions,
-            wifi_interface: base.wifi_interface.clone(),
-            ethernet_interface: base.ethernet_interface,
-            disk_guid,
-            db,
-            secret_store,
-            account: RwLock::new(account),
-            net_controller,
-            managers,
-            metrics_cache,
-            shutdown,
-            tor_socks: tor_proxy,
-            notification_manager,
-            open_authed_websockets: Mutex::new(BTreeMap::new()),
-            rpc_stream_continuations: Mutex::new(BTreeMap::new()),
-            wifi_manager: base
-                .wifi_interface
-                .map(|i| Arc::new(RwLock::new(WpaCli::init(i)))),
-            current_secret: Arc::new(
-                Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).map_err(|e| {
-                    tracing::debug!("{:?}", e);
-                    tracing::error!("Couldn't generate ec key");
-                    Error::new(
-                        color_eyre::eyre::eyre!("Couldn't generate ec key"),
-                        crate::ErrorKind::Unknown,
-                    )
-                })?,
-            ),
-            client: Client::builder()
-                .proxy(Proxy::custom(move |url| {
-                    if url.host_str().map_or(false, |h| h.ends_with(".onion")) {
-                        Some(tor_proxy_url.clone())
-                    } else {
-                        None
-                    }
-                }))
-                .build()
-                .with_kind(crate::ErrorKind::ParseUrl)?,
-            hardware: Hardware { devices, ram },
-            start_time: Instant::now(),
-        });
-
-        let res = Self(seed.clone());
-        res.cleanup_and_initialize().await?;
-        tracing::info!("Cleaned up transient states");
-        Ok(res)
-    }
-
-    #[instrument(skip_all)]
-    pub async fn shutdown(self) -> Result<(), Error> {
-        self.managers.empty().await?;
-        self.secret_store.close().await;
-        self.is_closed.store(true, Ordering::SeqCst);
-        tracing::info!("RPC Context is shutdown");
-        // TODO: shutdown http servers
-        Ok(())
-    }
-
-    #[instrument(skip(self))]
-    pub async fn cleanup_and_initialize(&self) -> Result<(), Error> {
-        self.db
-            .mutate(|f| {
-                let mut current_dependents = f
-                    .as_package_data()
-                    .keys()?
-                    .into_iter()
-                    .map(|k| (k.clone(), BTreeMap::new()))
-                    .collect::<BTreeMap<_, _>>();
-                for (package_id, package) in f.as_package_data_mut().as_entries_mut()? {
-                    for (k, v) in package
-                        .as_installed_mut()
-                        .into_iter()
-                        .flat_map(|i| i.clone().into_current_dependencies().into_entries())
-                        .flatten()
-                    {
-                        let mut entry: BTreeMap<_, _> =
-                            current_dependents.remove(&k).unwrap_or_default();
-                        entry.insert(package_id.clone(), v.de()?);
-                        current_dependents.insert(k, entry);
-                    }
-                }
-                for (package_id, current_dependents) in current_dependents {
-                    if let Some(deps) = f
-                        .as_package_data_mut()
-                        .as_idx_mut(&package_id)
-                        .and_then(|pde| pde.expect_as_installed_mut().ok())
-                        .map(|i| i.as_installed_mut().as_current_dependents_mut())
-                    {
-                        deps.ser(&CurrentDependents(current_dependents))?;
-                    } else if let Some(deps) = f
-                        .as_package_data_mut()
-                        .as_idx_mut(&package_id)
-                        .and_then(|pde| pde.expect_as_removing_mut().ok())
-                        .map(|i| i.as_removing_mut().as_current_dependents_mut())
-                    {
-                        deps.ser(&CurrentDependents(current_dependents))?;
-                    }
-                }
-                Ok(())
-            })
-            .await?;
-
-        let peek = self.db.peek().await;
-
-        for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
-            let action = match package.as_match() {
-                PackageDataEntryMatchModelRef::Installing(_)
-                | PackageDataEntryMatchModelRef::Restoring(_)
-                | PackageDataEntryMatchModelRef::Updating(_) => {
-                    cleanup_failed(self, &package_id).await
-                }
-                PackageDataEntryMatchModelRef::Removing(_) => {
-                    uninstall(
-                        self,
-                        self.secret_store.acquire().await?.as_mut(),
-                        &package_id,
-                    )
-                    .await
-                }
-                PackageDataEntryMatchModelRef::Installed(m) => {
-                    let version = m.as_manifest().as_version().clone().de()?;
-                    let volumes = m.as_manifest().as_volumes().de()?;
-                    for (volume_id, volume_info) in &*volumes {
-                        let tmp_path = to_tmp_path(volume_info.path_for(
-                            &self.datadir,
-                            &package_id,
-                            &version,
-                            volume_id,
-                        ))
-                        .with_kind(ErrorKind::Filesystem)?;
-                        if tokio::fs::metadata(&tmp_path).await.is_ok() {
-                            tokio::fs::remove_dir_all(&tmp_path).await?;
-                        }
-                    }
-                    Ok(())
-                }
-                _ => continue,
-            };
-            if let Err(e) = action {
-                tracing::error!("Failed to clean up package {}: {}", package_id, e);
-                tracing::debug!("{:?}", e);
-            }
-        }
-        let peek = self
-            .db
-            .mutate(|v| {
-                for (_, pde) in v.as_package_data_mut().as_entries_mut()? {
-                    let status = pde
-                        .expect_as_installed_mut()?
-                        .as_installed_mut()
-                        .as_status_mut()
-                        .as_main_mut();
-                    let running = status.clone().de()?.running();
-                    status.ser(&if running {
-                        MainStatus::Starting
-                    } else {
-                        MainStatus::Stopped
-                    })?;
-                }
-                Ok(v.clone())
-            })
-            .await?;
-        self.managers.init(self.clone(), peek.clone()).await?;
-        tracing::info!("Initialized Package Managers");
-
-        let mut all_dependency_config_errs = BTreeMap::new();
-        for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
-            let package = package.clone();
-            if let Some(current_dependencies) = package
-                .as_installed()
-                .and_then(|x| x.as_current_dependencies().de().ok())
-            {
-                let manifest = package.as_manifest().de()?;
-                all_dependency_config_errs.insert(
-                    package_id.clone(),
-                    compute_dependency_config_errs(
-                        self,
-                        &peek,
-                        &manifest,
-                        &current_dependencies,
-                        &Default::default(),
-                    )
-                    .await?,
-                );
-            }
-        }
-        self.db
-            .mutate(|v| {
-                for (package_id, errs) in all_dependency_config_errs {
-                    if let Some(config_errors) = v
-                        .as_package_data_mut()
-                        .as_idx_mut(&package_id)
-                        .and_then(|pde| pde.as_installed_mut())
-                        .map(|i| i.as_status_mut().as_dependency_config_errors_mut())
-                    {
-                        config_errors.ser(&errs)?;
-                    }
-                }
-                Ok(())
-            })
-            .await?;
-
-        Ok(())
-    }
-
-    #[instrument(skip_all)]
-    pub async fn clean_continuations(&self) {
-        let mut continuations = self.rpc_stream_continuations.lock().await;
-        let mut to_remove = Vec::new();
-        for (guid, cont) in &*continuations {
-            if cont.is_timed_out() {
-                to_remove.push(guid.clone());
-            }
-        }
-        for guid in to_remove {
-            continuations.remove(&guid);
-        }
-    }
-
-    #[instrument(skip_all)]
-    pub async fn add_continuation(&self, guid: RequestGuid, handler: RpcContinuation) {
-        self.clean_continuations().await;
-        self.rpc_stream_continuations
-            .lock()
-            .await
-            .insert(guid, handler);
-    }
-
-    pub async fn get_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
-        let mut continuations = self.rpc_stream_continuations.lock().await;
-        if let Some(cont) = continuations.remove(guid) {
-            cont.into_handler().await
-        } else {
-            None
-        }
-    }
-
-    pub async fn get_ws_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
-        let continuations = self.rpc_stream_continuations.lock().await;
-        if matches!(continuations.get(guid), Some(RpcContinuation::WebSocket(_))) {
-            drop(continuations);
-            self.get_continuation_handler(guid).await
-        } else {
-            None
-        }
-    }
-
-    pub async fn get_rest_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
-        let continuations = self.rpc_stream_continuations.lock().await;
-        if matches!(continuations.get(guid), Some(RpcContinuation::Rest(_))) {
-            drop(continuations);
-            self.get_continuation_handler(guid).await
-        } else {
-            None
-        }
-    }
-}
-impl AsRef<Jwk> for RpcContext {
-    fn as_ref(&self) -> &Jwk {
-        &CURRENT_SECRET
-    }
-}
-impl Context for RpcContext {}
-impl Deref for RpcContext {
-    type Target = RpcContextSeed;
-    fn deref(&self) -> &Self::Target {
-        #[cfg(feature = "unstable")]
-        if self.0.is_closed.load(Ordering::SeqCst) {
-            panic!(
-                "RpcContext used after shutdown! {}",
-                tracing_error::SpanTrace::capture()
-            );
-        }
-        &self.0
-    }
-}
-impl Drop for RpcContext {
-    fn drop(&mut self) {
-        #[cfg(feature = "unstable")]
-        if self.0.is_closed.load(Ordering::SeqCst) {
-            tracing::info!(
-                "RpcContext dropped. {} left.",
-                Arc::strong_count(&self.0) - 1
-            );
-        }
-    }
-}

backend/src/context/setup.rs

@@ -1,149 +0,0 @@
-use std::ops::Deref;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use josekit::jwk::Jwk;
-use patch_db::json_ptr::JsonPointer;
-use patch_db::PatchDb;
-use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::Context;
-use serde::{Deserialize, Serialize};
-use sqlx::postgres::PgConnectOptions;
-use sqlx::PgPool;
-use tokio::sync::broadcast::Sender;
-use tokio::sync::RwLock;
-use tracing::instrument;
-
-use crate::account::AccountInfo;
-use crate::db::model::Database;
-use crate::disk::OsPartitionInfo;
-use crate::init::init_postgres;
-use crate::setup::SetupStatus;
-use crate::util::config::load_config_from_paths;
-use crate::{Error, ResultExt};
-
-lazy_static::lazy_static! {
-    pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| {
-        tracing::debug!("{:?}", e);
-        tracing::error!("Couldn't generate ec key");
-        panic!("Couldn't generate ec key")
-    });
-}
-
-#[derive(Clone, Serialize, Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct SetupResult {
-    pub tor_address: String,
-    pub lan_address: String,
-    pub root_ca: String,
-}
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct SetupContextConfig {
-    pub os_partitions: OsPartitionInfo,
-    pub migration_batch_rows: Option<usize>,
-    pub migration_prefetch_rows: Option<usize>,
-    pub datadir: Option<PathBuf>,
-    #[serde(default)]
-    pub disable_encryption: bool,
-}
-impl SetupContextConfig {
-    #[instrument(skip_all)]
-    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
-        tokio::task::spawn_blocking(move || {
-            load_config_from_paths(
-                path.as_ref()
-                    .into_iter()
-                    .map(|p| p.as_ref())
-                    .chain(std::iter::once(Path::new(
-                        crate::util::config::DEVICE_CONFIG_PATH,
-                    )))
-                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
-            )
-        })
-        .await
-        .unwrap()
-    }
-    pub fn datadir(&self) -> &Path {
-        self.datadir
-            .as_deref()
-            .unwrap_or_else(|| Path::new("/embassy-data"))
-    }
-}
-
-pub struct SetupContextSeed {
-    pub os_partitions: OsPartitionInfo,
-    pub config_path: Option<PathBuf>,
-    pub migration_batch_rows: usize,
-    pub migration_prefetch_rows: usize,
-    pub disable_encryption: bool,
-    pub shutdown: Sender<()>,
-    pub datadir: PathBuf,
-    pub selected_v2_drive: RwLock<Option<PathBuf>>,
-    pub cached_product_key: RwLock<Option<Arc<String>>>,
-    pub setup_status: RwLock<Option<Result<SetupStatus, RpcError>>>,
-    pub setup_result: RwLock<Option<(Arc<String>, SetupResult)>>,
-}
-
-impl AsRef<Jwk> for SetupContextSeed {
-    fn as_ref(&self) -> &Jwk {
-        &*CURRENT_SECRET
-    }
-}
-
-#[derive(Clone)]
-pub struct SetupContext(Arc<SetupContextSeed>);
-impl SetupContext {
-    #[instrument(skip_all)]
-    pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
-        let cfg = SetupContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
-        let (shutdown, _) = tokio::sync::broadcast::channel(1);
-        let datadir = cfg.datadir().to_owned();
-        Ok(Self(Arc::new(SetupContextSeed {
-            os_partitions: cfg.os_partitions,
-            config_path: path.as_ref().map(|p| p.as_ref().to_owned()),
-            migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000),
-            migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000),
-            disable_encryption: cfg.disable_encryption,
-            shutdown,
-            datadir,
-            selected_v2_drive: RwLock::new(None),
-            cached_product_key: RwLock::new(None),
-            setup_status: RwLock::new(None),
-            setup_result: RwLock::new(None),
-        })))
-    }
-    #[instrument(skip_all)]
-    pub async fn db(&self, account: &AccountInfo) -> Result<PatchDb, Error> {
-        let db_path = self.datadir.join("main").join("embassy.db");
-        let db = PatchDb::open(&db_path)
-            .await
-            .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
-        if !db.exists(&<JsonPointer>::default()).await {
-            db.put(&<JsonPointer>::default(), &Database::init(account))
-                .await?;
-        }
-        Ok(db)
-    }
-    #[instrument(skip_all)]
-    pub async fn secret_store(&self) -> Result<PgPool, Error> {
-        init_postgres(&self.datadir).await?;
-        let secret_store =
-            PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
-                .await?;
-        sqlx::migrate!()
-            .run(&secret_store)
-            .await
-            .with_kind(crate::ErrorKind::Database)?;
-        Ok(secret_store)
-    }
-}
-
-impl Context for SetupContext {}
-impl Deref for SetupContext {
-    type Target = SetupContextSeed;
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}

backend/src/control.rs

@@ -1,92 +0,0 @@
-use color_eyre::eyre::eyre;
-use rpc_toolkit::command;
-use tracing::instrument;
-
-use crate::context::RpcContext;
-use crate::prelude::*;
-use crate::s9pk::manifest::PackageId;
-use crate::status::MainStatus;
-use crate::util::display_none;
-use crate::Error;
-
-#[command(display(display_none), metadata(sync_db = true))]
-#[instrument(skip_all)]
-pub async fn start(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
-    let peek = ctx.db.peek().await;
-    let version = peek
-        .as_package_data()
-        .as_idx(&id)
-        .or_not_found(&id)?
-        .as_installed()
-        .or_not_found(&id)?
-        .as_manifest()
-        .as_version()
-        .de()?;
-
-    ctx.managers
-        .get(&(id, version))
-        .await
-        .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
-        .start()
-        .await;
-
-    Ok(())
-}
-
-#[command(display(display_none), metadata(sync_db = true))]
-pub async fn stop(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<MainStatus, Error> {
-    let peek = ctx.db.peek().await;
-    let version = peek
-        .as_package_data()
-        .as_idx(&id)
-        .or_not_found(&id)?
-        .as_installed()
-        .or_not_found(&id)?
-        .as_manifest()
-        .as_version()
-        .de()?;
-
-    let last_statuts = ctx
-        .db
-        .mutate(|v| {
-            v.as_package_data_mut()
-                .as_idx_mut(&id)
-                .and_then(|x| x.as_installed_mut())
-                .ok_or_else(|| Error::new(eyre!("{} is not installed", id), ErrorKind::NotFound))?
-                .as_status_mut()
-                .as_main_mut()
-                .replace(&MainStatus::Stopping)
-        })
-        .await?;
-
-    ctx.managers
-        .get(&(id, version))
-        .await
-        .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
-        .stop()
-        .await;
-
-    Ok(last_statuts)
-}
-
-#[command(display(display_none), metadata(sync_db = true))]
-pub async fn restart(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
-    let peek = ctx.db.peek().await;
-    let version = peek
-        .as_package_data()
-        .as_idx(&id)
-        .or_not_found(&id)?
-        .expect_as_installed()?
-        .as_manifest()
-        .as_version()
-        .de()?;
-
-    ctx.managers
-        .get(&(id, version))
-        .await
-        .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
-        .restart()
-        .await;
-
-    Ok(())
-}

backend/src/core/mod.rs

@@ -1 +0,0 @@
-pub mod rpc_continuations;

backend/src/core/rpc_continuations.rs

@@ -1,116 +0,0 @@
-use std::sync::Arc;
-use std::time::Duration;
-
-use futures::future::BoxFuture;
-use futures::FutureExt;
-use helpers::TimedResource;
-use hyper::upgrade::Upgraded;
-use hyper::{Body, Error as HyperError, Request, Response};
-use rand::RngCore;
-use tokio::task::JoinError;
-use tokio_tungstenite::WebSocketStream;
-
-use crate::{Error, ResultExt};
-
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
-pub struct RequestGuid<T: AsRef<str> = String>(Arc<T>);
-impl RequestGuid {
-    pub fn new() -> Self {
-        let mut buf = [0; 40];
-        rand::thread_rng().fill_bytes(&mut buf);
-        RequestGuid(Arc::new(base32::encode(
-            base32::Alphabet::RFC4648 { padding: false },
-            &buf,
-        )))
-    }
-
-    pub fn from(r: &str) -> Option<RequestGuid> {
-        if r.len() != 64 {
-            return None;
-        }
-        for c in r.chars() {
-            if !(c >= 'A' && c <= 'Z' || c >= '2' && c <= '7') {
-                return None;
-            }
-        }
-        Some(RequestGuid(Arc::new(r.to_owned())))
-    }
-}
-#[test]
-fn parse_guid() {
-    println!(
-        "{:?}",
-        RequestGuid::from(&format!("{}", RequestGuid::new()))
-    )
-}
-
-impl<T: AsRef<str>> std::fmt::Display for RequestGuid<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        (&*self.0).as_ref().fmt(f)
-    }
-}
-
-pub type RestHandler = Box<
-    dyn FnOnce(Request<Body>) -> BoxFuture<'static, Result<Response<Body>, crate::Error>> + Send,
->;
-
-pub type WebSocketHandler = Box<
-    dyn FnOnce(
-            BoxFuture<'static, Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
-        ) -> BoxFuture<'static, Result<(), Error>>
-        + Send,
->;
-
-pub enum RpcContinuation {
-    Rest(TimedResource<RestHandler>),
-    WebSocket(TimedResource<WebSocketHandler>),
-}
-impl RpcContinuation {
-    pub fn rest(handler: RestHandler, timeout: Duration) -> Self {
-        RpcContinuation::Rest(TimedResource::new(handler, timeout))
-    }
-    pub fn ws(handler: WebSocketHandler, timeout: Duration) -> Self {
-        RpcContinuation::WebSocket(TimedResource::new(handler, timeout))
-    }
-    pub fn is_timed_out(&self) -> bool {
-        match self {
-            RpcContinuation::Rest(a) => a.is_timed_out(),
-            RpcContinuation::WebSocket(a) => a.is_timed_out(),
-        }
-    }
-    pub async fn into_handler(self) -> Option<RestHandler> {
-        match self {
-            RpcContinuation::Rest(handler) => handler.get().await,
-            RpcContinuation::WebSocket(handler) => {
-                if let Some(handler) = handler.get().await {
-                    Some(Box::new(
-                        |req: Request<Body>| -> BoxFuture<'static, Result<Response<Body>, Error>> {
-                            async move {
-                                let (parts, body) = req.into_parts();
-                                let req = Request::from_parts(parts, body);
-                                let (res, ws_fut) = hyper_ws_listener::create_ws(req)
-                                    .with_kind(crate::ErrorKind::Network)?;
-                                if let Some(ws_fut) = ws_fut {
-                                    tokio::task::spawn(async move {
-                                        match handler(ws_fut.boxed()).await {
-                                            Ok(()) => (),
-                                            Err(e) => {
-                                                tracing::error!("WebSocket Closed: {}", e);
-                                                tracing::debug!("{:?}", e);
-                                            }
-                                        }
-                                    });
-                                }
-
-                                Ok(res)
-                            }
-                            .boxed()
-                        },
-                    ))
-                } else {
-                    None
-                }
-            }
-        }
-    }
-}

backend/src/db/mod.rs

@@ -1,370 +0,0 @@
-pub mod model;
-pub mod package;
-pub mod prelude;
-
-use std::future::Future;
-use std::path::PathBuf;
-use std::sync::Arc;
-
-use futures::{FutureExt, SinkExt, StreamExt};
-use patch_db::json_ptr::JsonPointer;
-use patch_db::{Dump, Revision};
-use rpc_toolkit::command;
-use rpc_toolkit::hyper::upgrade::Upgraded;
-use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response};
-use rpc_toolkit::yajrc::RpcError;
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use tokio::sync::oneshot;
-use tokio::task::JoinError;
-use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
-use tokio_tungstenite::tungstenite::protocol::CloseFrame;
-use tokio_tungstenite::tungstenite::Message;
-use tokio_tungstenite::WebSocketStream;
-use tracing::instrument;
-
-use crate::context::{CliContext, RpcContext};
-use crate::middleware::auth::{HasValidSession, HashSessionToken};
-use crate::prelude::*;
-use crate::util::display_none;
-use crate::util::serde::{display_serializable, IoFormat};
-
-#[instrument(skip_all)]
-async fn ws_handler<
-    WSFut: Future<Output = Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
->(
-    ctx: RpcContext,
-    session: Option<(HasValidSession, HashSessionToken)>,
-    ws_fut: WSFut,
-) -> Result<(), Error> {
-    let (dump, sub) = ctx.db.dump_and_sub().await;
-    let mut stream = ws_fut
-        .await
-        .with_kind(ErrorKind::Network)?
-        .with_kind(ErrorKind::Unknown)?;
-
-    if let Some((session, token)) = session {
-        let kill = subscribe_to_session_kill(&ctx, token).await;
-        send_dump(session, &mut stream, dump).await?;
-
-        deal_with_messages(session, kill, sub, stream).await?;
-    } else {
-        stream
-            .close(Some(CloseFrame {
-                code: CloseCode::Error,
-                reason: "UNAUTHORIZED".into(),
-            }))
-            .await
-            .with_kind(ErrorKind::Network)?;
-    }
-
-    Ok(())
-}
-
-async fn subscribe_to_session_kill(
-    ctx: &RpcContext,
-    token: HashSessionToken,
-) -> oneshot::Receiver<()> {
-    let (send, recv) = oneshot::channel();
-    let mut guard = ctx.open_authed_websockets.lock().await;
-    if !guard.contains_key(&token) {
-        guard.insert(token, vec![send]);
-    } else {
-        guard.get_mut(&token).unwrap().push(send);
-    }
-    recv
-}
-
-#[instrument(skip_all)]
-async fn deal_with_messages(
-    _has_valid_authentication: HasValidSession,
-    mut kill: oneshot::Receiver<()>,
-    mut sub: patch_db::Subscriber,
-    mut stream: WebSocketStream<Upgraded>,
-) -> Result<(), Error> {
-    let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(5));
-
-    loop {
-        futures::select! {
-            _ = (&mut kill).fuse() => {
-                tracing::info!("Closing WebSocket: Reason: Session Terminated");
-                stream
-                    .close(Some(CloseFrame {
-                        code: CloseCode::Error,
-                        reason: "UNAUTHORIZED".into(),
-                    }))
-                    .await
-                    .with_kind(ErrorKind::Network)?;
-                return Ok(())
-            }
-            new_rev = sub.recv().fuse() => {
-                let rev = new_rev.expect("UNREACHABLE: patch-db is dropped");
-                stream
-                    .send(Message::Text(serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?))
-                    .await
-                    .with_kind(ErrorKind::Network)?;
-            }
-            message = stream.next().fuse() => {
-                let message = message.transpose().with_kind(ErrorKind::Network)?;
-                match message {
-                    None => {
-                        tracing::info!("Closing WebSocket: Stream Finished");
-                        return Ok(())
-                    }
-                    _ => (),
-                }
-            }
-            // This is trying to give a health checks to the home to keep the ui alive.
-            _ = timer.tick().fuse() => {
-                stream
-                    .send(Message::Ping(vec![]))
-                    .await
-                    .with_kind(crate::ErrorKind::Network)?;
-            }
-        }
-    }
-}
-
-async fn send_dump(
-    _has_valid_authentication: HasValidSession,
-    stream: &mut WebSocketStream<Upgraded>,
-    dump: Dump,
-) -> Result<(), Error> {
-    stream
-        .send(Message::Text(
-            serde_json::to_string(&dump).with_kind(ErrorKind::Serialization)?,
-        ))
-        .await
-        .with_kind(ErrorKind::Network)?;
-    Ok(())
-}
-
-pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<Body>, Error> {
-    let (parts, body) = req.into_parts();
-    let session = match async {
-        let token = HashSessionToken::from_request_parts(&parts)?;
-        let session = HasValidSession::from_request_parts(&parts, &ctx).await?;
-        Ok::<_, Error>((session, token))
-    }
-    .await
-    {
-        Ok(a) => Some(a),
-        Err(e) => {
-            if e.kind != ErrorKind::Authorization {
-                tracing::error!("Error Authenticating Websocket: {}", e);
-                tracing::debug!("{:?}", e);
-            }
-            None
-        }
-    };
-    let req = Request::from_parts(parts, body);
-    let (res, ws_fut) = hyper_ws_listener::create_ws(req).with_kind(ErrorKind::Network)?;
-    if let Some(ws_fut) = ws_fut {
-        tokio::task::spawn(async move {
-            match ws_handler(ctx, session, ws_fut).await {
-                Ok(()) => (),
-                Err(e) => {
-                    tracing::error!("WebSocket Closed: {}", e);
-                    tracing::debug!("{:?}", e);
-                }
-            }
-        });
-    }
-
-    Ok(res)
-}
-
-#[command(subcommands(dump, put, apply))]
-pub fn db() -> Result<(), RpcError> {
-    Ok(())
-}
-
-#[derive(Deserialize, Serialize)]
-#[serde(untagged)]
-pub enum RevisionsRes {
-    Revisions(Vec<Arc<Revision>>),
-    Dump(Dump),
-}
-
-#[instrument(skip_all)]
-async fn cli_dump(
-    ctx: CliContext,
-    _format: Option<IoFormat>,
-    path: Option<PathBuf>,
-) -> Result<Dump, RpcError> {
-    let dump = if let Some(path) = path {
-        PatchDb::open(path).await?.dump().await
-    } else {
-        rpc_toolkit::command_helpers::call_remote(
-            ctx,
-            "db.dump",
-            serde_json::json!({}),
-            std::marker::PhantomData::<Dump>,
-        )
-        .await?
-        .result?
-    };
-
-    Ok(dump)
-}
-
-#[command(
-    custom_cli(cli_dump(async, context(CliContext))),
-    display(display_serializable)
-)]
-pub async fn dump(
-    #[context] ctx: RpcContext,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-    #[allow(unused_variables)]
-    #[arg]
-    path: Option<PathBuf>,
-) -> Result<Dump, Error> {
-    Ok(ctx.db.dump().await)
-}
-
-fn apply_expr(input: jaq_core::Val, expr: &str) -> Result<jaq_core::Val, Error> {
-    let (expr, errs) = jaq_core::parse::parse(expr, jaq_core::parse::main());
-
-    let Some(expr) = expr else {
-        return Err(Error::new(
-            eyre!("Failed to parse expression: {:?}", errs),
-            crate::ErrorKind::InvalidRequest,
-        ));
-    };
-
-    let mut errs = Vec::new();
-
-    let mut defs = jaq_core::Definitions::core();
-    for def in jaq_std::std() {
-        defs.insert(def, &mut errs);
-    }
-
-    let filter = defs.finish(expr, Vec::new(), &mut errs);
-
-    if !errs.is_empty() {
-        return Err(Error::new(
-            eyre!("Failed to compile expression: {:?}", errs),
-            crate::ErrorKind::InvalidRequest,
-        ));
-    };
-
-    let inputs = jaq_core::RcIter::new(std::iter::empty());
-    let mut res_iter = filter.run(jaq_core::Ctx::new([], &inputs), input);
-
-    let Some(res) = res_iter
-        .next()
-        .transpose()
-        .map_err(|e| eyre!("{e}"))
-        .with_kind(crate::ErrorKind::Deserialization)?
-    else {
-        return Err(Error::new(
-            eyre!("expr returned no results"),
-            crate::ErrorKind::InvalidRequest,
-        ));
-    };
-
-    if res_iter.next().is_some() {
-        return Err(Error::new(
-            eyre!("expr returned too many results"),
-            crate::ErrorKind::InvalidRequest,
-        ));
-    }
-
-    Ok(res)
-}
-
-#[instrument(skip_all)]
-async fn cli_apply(ctx: CliContext, expr: String, path: Option<PathBuf>) -> Result<(), RpcError> {
-    if let Some(path) = path {
-        PatchDb::open(path)
-            .await?
-            .mutate(|db| {
-                let res = apply_expr(
-                    serde_json::to_value(patch_db::Value::from(db.clone()))
-                        .with_kind(ErrorKind::Deserialization)?
-                        .into(),
-                    &expr,
-                )?;
-
-                db.ser(
-                    &serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(
-                        |_| {
-                            (
-                                crate::ErrorKind::Deserialization,
-                                "result does not match database model",
-                            )
-                        },
-                    )?,
-                )
-            })
-            .await?;
-    } else {
-        rpc_toolkit::command_helpers::call_remote(
-            ctx,
-            "db.apply",
-            serde_json::json!({ "expr": expr }),
-            std::marker::PhantomData::<()>,
-        )
-        .await?
-        .result?;
-    }
-
-    Ok(())
-}
-
-#[command(
-    custom_cli(cli_apply(async, context(CliContext))),
-    display(display_none)
-)]
-pub async fn apply(
-    #[context] ctx: RpcContext,
-    #[arg] expr: String,
-    #[allow(unused_variables)]
-    #[arg]
-    path: Option<PathBuf>,
-) -> Result<(), Error> {
-    ctx.db
-        .mutate(|db| {
-            let res = apply_expr(
-                serde_json::to_value(patch_db::Value::from(db.clone()))
-                    .with_kind(ErrorKind::Deserialization)?
-                    .into(),
-                &expr,
-            )?;
-
-            db.ser(
-                &serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(|_| {
-                    (
-                        crate::ErrorKind::Deserialization,
-                        "result does not match database model",
-                    )
-                })?,
-            )
-        })
-        .await
-}
-
-#[command(subcommands(ui))]
-pub fn put() -> Result<(), RpcError> {
-    Ok(())
-}
-
-#[command(display(display_serializable))]
-#[instrument(skip_all)]
-pub async fn ui(
-    #[context] ctx: RpcContext,
-    #[arg] pointer: JsonPointer,
-    #[arg] value: Value,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-) -> Result<(), Error> {
-    let ptr = "/ui"
-        .parse::<JsonPointer>()
-        .with_kind(ErrorKind::Database)?
-        + &pointer;
-    ctx.db.put(&ptr, &value).await?;
-    Ok(())
-}

backend/src/db/model.rs

@@ -1,524 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet};
-use std::net::{Ipv4Addr, Ipv6Addr};
-use std::sync::Arc;
-
-use chrono::{DateTime, Utc};
-use emver::VersionRange;
-use imbl_value::InternedString;
-use ipnet::{Ipv4Net, Ipv6Net};
-use isocountry::CountryCode;
-use itertools::Itertools;
-use models::{DataUrl, HealthCheckId, InterfaceId};
-use openssl::hash::MessageDigest;
-use patch_db::{HasModel, Value};
-use reqwest::Url;
-use serde::{Deserialize, Serialize};
-use ssh_key::public::Ed25519PublicKey;
-
-use crate::account::AccountInfo;
-use crate::config::spec::PackagePointerSpec;
-use crate::install::progress::InstallProgress;
-use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
-use crate::prelude::*;
-use crate::s9pk::manifest::{Manifest, PackageId};
-use crate::status::Status;
-use crate::util::Version;
-use crate::version::{Current, VersionT};
-use crate::{ARCH, PLATFORM};
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-// #[macro_debug]
-pub struct Database {
-    pub server_info: ServerInfo,
-    pub package_data: AllPackageData,
-    pub ui: Value,
-}
-impl Database {
-    pub fn init(account: &AccountInfo) -> Self {
-        let lan_address = account.hostname.lan_address().parse().unwrap();
-        Database {
-            server_info: ServerInfo {
-                arch: get_arch(),
-                platform: get_platform(),
-                id: account.server_id.clone(),
-                version: Current::new().semver().into(),
-                hostname: account.hostname.no_dot_host_name(),
-                last_backup: None,
-                last_wifi_region: None,
-                eos_version_compat: Current::new().compat().clone(),
-                lan_address,
-                tor_address: format!("https://{}", account.key.tor_address())
-                    .parse()
-                    .unwrap(),
-                ip_info: BTreeMap::new(),
-                status_info: ServerStatus {
-                    backup_progress: None,
-                    updated: false,
-                    update_progress: None,
-                    shutting_down: false,
-                    restarting: false,
-                },
-                wifi: WifiInfo {
-                    ssids: Vec::new(),
-                    connected: None,
-                    selected: None,
-                },
-                unread_notification_count: 0,
-                connection_addresses: ConnectionAddresses {
-                    tor: Vec::new(),
-                    clearnet: Vec::new(),
-                },
-                password_hash: account.password.clone(),
-                pubkey: ssh_key::PublicKey::from(Ed25519PublicKey::from(&account.key.ssh_key()))
-                    .to_openssh()
-                    .unwrap(),
-                ca_fingerprint: account
-                    .root_ca_cert
-                    .digest(MessageDigest::sha256())
-                    .unwrap()
-                    .iter()
-                    .map(|x| format!("{x:X}"))
-                    .join(":"),
-                ntp_synced: false,
-                zram: true,
-            },
-            package_data: AllPackageData::default(),
-            ui: serde_json::from_str(include_str!("../../../frontend/patchdb-ui-seed.json"))
-                .unwrap(),
-        }
-    }
-}
-
-pub type DatabaseModel = Model<Database>;
-
-fn get_arch() -> InternedString {
-    (*ARCH).into()
-}
-
-fn get_platform() -> InternedString {
-    (&*PLATFORM).into()
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct ServerInfo {
-    #[serde(default = "get_arch")]
-    pub arch: InternedString,
-    #[serde(default = "get_platform")]
-    pub platform: InternedString,
-    pub id: String,
-    pub hostname: String,
-    pub version: Version,
-    pub last_backup: Option<DateTime<Utc>>,
-    /// Used in the wifi to determine the region to set the system to
-    pub last_wifi_region: Option<CountryCode>,
-    pub eos_version_compat: VersionRange,
-    pub lan_address: Url,
-    pub tor_address: Url,
-    pub ip_info: BTreeMap<String, IpInfo>,
-    #[serde(default)]
-    pub status_info: ServerStatus,
-    pub wifi: WifiInfo,
-    pub unread_notification_count: u64,
-    pub connection_addresses: ConnectionAddresses,
-    pub password_hash: String,
-    pub pubkey: String,
-    pub ca_fingerprint: String,
-    #[serde(default)]
-    pub ntp_synced: bool,
-    #[serde(default)]
-    pub zram: bool,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct IpInfo {
-    pub ipv4_range: Option<Ipv4Net>,
-    pub ipv4: Option<Ipv4Addr>,
-    pub ipv6_range: Option<Ipv6Net>,
-    pub ipv6: Option<Ipv6Addr>,
-}
-impl IpInfo {
-    pub async fn for_interface(iface: &str) -> Result<Self, Error> {
-        let (ipv4, ipv4_range) = get_iface_ipv4_addr(iface).await?.unzip();
-        let (ipv6, ipv6_range) = get_iface_ipv6_addr(iface).await?.unzip();
-        Ok(Self {
-            ipv4_range,
-            ipv4,
-            ipv6_range,
-            ipv6,
-        })
-    }
-}
-
-#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
-#[model = "Model<Self>"]
-pub struct BackupProgress {
-    pub complete: bool,
-}
-
-#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct ServerStatus {
-    pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
-    pub updated: bool,
-    pub update_progress: Option<UpdateProgress>,
-    #[serde(default)]
-    pub shutting_down: bool,
-    #[serde(default)]
-    pub restarting: bool,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct UpdateProgress {
-    pub size: Option<u64>,
-    pub downloaded: u64,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct WifiInfo {
-    pub ssids: Vec<String>,
-    pub selected: Option<String>,
-    pub connected: Option<String>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct ServerSpecs {
-    pub cpu: String,
-    pub disk: String,
-    pub memory: String,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct ConnectionAddresses {
-    pub tor: Vec<String>,
-    pub clearnet: Vec<String>,
-}
-
-#[derive(Debug, Default, Deserialize, Serialize)]
-pub struct AllPackageData(pub BTreeMap<PackageId, PackageDataEntry>);
-impl Map for AllPackageData {
-    type Key = PackageId;
-    type Value = PackageDataEntry;
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct StaticFiles {
-    license: String,
-    instructions: String,
-    icon: String,
-}
-impl StaticFiles {
-    pub fn local(id: &PackageId, version: &Version, icon_type: &str) -> Self {
-        StaticFiles {
-            license: format!("/public/package-data/{}/{}/LICENSE.md", id, version),
-            instructions: format!("/public/package-data/{}/{}/INSTRUCTIONS.md", id, version),
-            icon: format!("/public/package-data/{}/{}/icon.{}", id, version, icon_type),
-        }
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryInstalling {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub install_progress: Arc<InstallProgress>,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryUpdating {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub installed: InstalledPackageInfo,
-    pub install_progress: Arc<InstallProgress>,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryRestoring {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub install_progress: Arc<InstallProgress>,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryRemoving {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub removing: InstalledPackageInfo,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryInstalled {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub installed: InstalledPackageInfo,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(tag = "state")]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-// #[macro_debug]
-pub enum PackageDataEntry {
-    Installing(PackageDataEntryInstalling),
-    Updating(PackageDataEntryUpdating),
-    Restoring(PackageDataEntryRestoring),
-    Removing(PackageDataEntryRemoving),
-    Installed(PackageDataEntryInstalled),
-}
-impl Model<PackageDataEntry> {
-    pub fn expect_into_installed(self) -> Result<Model<PackageDataEntryInstalled>, Error> {
-        if let PackageDataEntryMatchModel::Installed(a) = self.into_match() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in installed state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_installed(&self) -> Result<&Model<PackageDataEntryInstalled>, Error> {
-        if let PackageDataEntryMatchModelRef::Installed(a) = self.as_match() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in installed state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_installed_mut(
-        &mut self,
-    ) -> Result<&mut Model<PackageDataEntryInstalled>, Error> {
-        if let PackageDataEntryMatchModelMut::Installed(a) = self.as_match_mut() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in installed state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_into_removing(self) -> Result<Model<PackageDataEntryRemoving>, Error> {
-        if let PackageDataEntryMatchModel::Removing(a) = self.into_match() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in removing state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_removing(&self) -> Result<&Model<PackageDataEntryRemoving>, Error> {
-        if let PackageDataEntryMatchModelRef::Removing(a) = self.as_match() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in removing state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_removing_mut(
-        &mut self,
-    ) -> Result<&mut Model<PackageDataEntryRemoving>, Error> {
-        if let PackageDataEntryMatchModelMut::Removing(a) = self.as_match_mut() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in removing state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_installing_mut(
-        &mut self,
-    ) -> Result<&mut Model<PackageDataEntryInstalling>, Error> {
-        if let PackageDataEntryMatchModelMut::Installing(a) = self.as_match_mut() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in installing state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn into_manifest(self) -> Model<Manifest> {
-        match self.into_match() {
-            PackageDataEntryMatchModel::Installing(a) => a.into_manifest(),
-            PackageDataEntryMatchModel::Updating(a) => a.into_installed().into_manifest(),
-            PackageDataEntryMatchModel::Restoring(a) => a.into_manifest(),
-            PackageDataEntryMatchModel::Removing(a) => a.into_manifest(),
-            PackageDataEntryMatchModel::Installed(a) => a.into_manifest(),
-            PackageDataEntryMatchModel::Error(_) => Model::from(Value::Null),
-        }
-    }
-    pub fn as_manifest(&self) -> &Model<Manifest> {
-        match self.as_match() {
-            PackageDataEntryMatchModelRef::Installing(a) => a.as_manifest(),
-            PackageDataEntryMatchModelRef::Updating(a) => a.as_installed().as_manifest(),
-            PackageDataEntryMatchModelRef::Restoring(a) => a.as_manifest(),
-            PackageDataEntryMatchModelRef::Removing(a) => a.as_manifest(),
-            PackageDataEntryMatchModelRef::Installed(a) => a.as_manifest(),
-            PackageDataEntryMatchModelRef::Error(_) => (&Value::Null).into(),
-        }
-    }
-    pub fn into_installed(self) -> Option<Model<InstalledPackageInfo>> {
-        match self.into_match() {
-            PackageDataEntryMatchModel::Installing(_) => None,
-            PackageDataEntryMatchModel::Updating(a) => Some(a.into_installed()),
-            PackageDataEntryMatchModel::Restoring(_) => None,
-            PackageDataEntryMatchModel::Removing(_) => None,
-            PackageDataEntryMatchModel::Installed(a) => Some(a.into_installed()),
-            PackageDataEntryMatchModel::Error(_) => None,
-        }
-    }
-    pub fn as_installed(&self) -> Option<&Model<InstalledPackageInfo>> {
-        match self.as_match() {
-            PackageDataEntryMatchModelRef::Installing(_) => None,
-            PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_installed()),
-            PackageDataEntryMatchModelRef::Restoring(_) => None,
-            PackageDataEntryMatchModelRef::Removing(_) => None,
-            PackageDataEntryMatchModelRef::Installed(a) => Some(a.as_installed()),
-            PackageDataEntryMatchModelRef::Error(_) => None,
-        }
-    }
-    pub fn as_installed_mut(&mut self) -> Option<&mut Model<InstalledPackageInfo>> {
-        match self.as_match_mut() {
-            PackageDataEntryMatchModelMut::Installing(_) => None,
-            PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_installed_mut()),
-            PackageDataEntryMatchModelMut::Restoring(_) => None,
-            PackageDataEntryMatchModelMut::Removing(_) => None,
-            PackageDataEntryMatchModelMut::Installed(a) => Some(a.as_installed_mut()),
-            PackageDataEntryMatchModelMut::Error(_) => None,
-        }
-    }
-    pub fn as_install_progress(&self) -> Option<&Model<Arc<InstallProgress>>> {
-        match self.as_match() {
-            PackageDataEntryMatchModelRef::Installing(a) => Some(a.as_install_progress()),
-            PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_install_progress()),
-            PackageDataEntryMatchModelRef::Restoring(a) => Some(a.as_install_progress()),
-            PackageDataEntryMatchModelRef::Removing(_) => None,
-            PackageDataEntryMatchModelRef::Installed(_) => None,
-            PackageDataEntryMatchModelRef::Error(_) => None,
-        }
-    }
-    pub fn as_install_progress_mut(&mut self) -> Option<&mut Model<Arc<InstallProgress>>> {
-        match self.as_match_mut() {
-            PackageDataEntryMatchModelMut::Installing(a) => Some(a.as_install_progress_mut()),
-            PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_install_progress_mut()),
-            PackageDataEntryMatchModelMut::Restoring(a) => Some(a.as_install_progress_mut()),
-            PackageDataEntryMatchModelMut::Removing(_) => None,
-            PackageDataEntryMatchModelMut::Installed(_) => None,
-            PackageDataEntryMatchModelMut::Error(_) => None,
-        }
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct InstalledPackageInfo {
-    pub status: Status,
-    pub marketplace_url: Option<Url>,
-    #[serde(default)]
-    #[serde(with = "crate::util::serde::ed25519_pubkey")]
-    pub developer_key: ed25519_dalek::VerifyingKey,
-    pub manifest: Manifest,
-    pub last_backup: Option<DateTime<Utc>>,
-    pub dependency_info: BTreeMap<PackageId, StaticDependencyInfo>,
-    pub current_dependents: CurrentDependents,
-    pub current_dependencies: CurrentDependencies,
-    pub interface_addresses: InterfaceAddressMap,
-}
-
-#[derive(Debug, Clone, Default, Deserialize, Serialize)]
-pub struct CurrentDependents(pub BTreeMap<PackageId, CurrentDependencyInfo>);
-impl CurrentDependents {
-    pub fn map(
-        mut self,
-        transform: impl Fn(
-            BTreeMap<PackageId, CurrentDependencyInfo>,
-        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
-    ) -> Self {
-        self.0 = transform(self.0);
-        self
-    }
-}
-impl Map for CurrentDependents {
-    type Key = PackageId;
-    type Value = CurrentDependencyInfo;
-}
-
-#[derive(Debug, Clone, Default, Deserialize, Serialize)]
-pub struct CurrentDependencies(pub BTreeMap<PackageId, CurrentDependencyInfo>);
-impl CurrentDependencies {
-    pub fn map(
-        mut self,
-        transform: impl Fn(
-            BTreeMap<PackageId, CurrentDependencyInfo>,
-        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
-    ) -> Self {
-        self.0 = transform(self.0);
-        self
-    }
-}
-impl Map for CurrentDependencies {
-    type Key = PackageId;
-    type Value = CurrentDependencyInfo;
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct StaticDependencyInfo {
-    pub title: String,
-    pub icon: DataUrl<'static>,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct CurrentDependencyInfo {
-    #[serde(default)]
-    pub pointers: BTreeSet<PackagePointerSpec>,
-    pub health_checks: BTreeSet<HealthCheckId>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct InterfaceAddressMap(pub BTreeMap<InterfaceId, InterfaceAddresses>);
-impl Map for InterfaceAddressMap {
-    type Key = InterfaceId;
-    type Value = InterfaceAddresses;
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct InterfaceAddresses {
-    pub tor_address: Option<String>,
-    pub lan_address: Option<String>,
-}

backend/src/db/package.rs

@@ -1,22 +0,0 @@
-use models::Version;
-
-use crate::prelude::*;
-use crate::s9pk::manifest::PackageId;
-
-pub fn get_packages(db: Peeked) -> Result<Vec<(PackageId, Version)>, Error> {
-    Ok(db
-        .as_package_data()
-        .keys()?
-        .into_iter()
-        .flat_map(|package_id| {
-            let version = db
-                .as_package_data()
-                .as_idx(&package_id)?
-                .as_manifest()
-                .as_version()
-                .de()
-                .ok()?;
-            Some((package_id, version))
-        })
-        .collect())
-}

backend/src/dependencies.rs

@@ -1,363 +0,0 @@
-use std::collections::BTreeMap;
-use std::time::Duration;
-
-use color_eyre::eyre::eyre;
-use emver::VersionRange;
-use models::OptionExt;
-use rand::SeedableRng;
-use rpc_toolkit::command;
-use serde::{Deserialize, Serialize};
-use tracing::instrument;
-
-use crate::config::action::ConfigRes;
-use crate::config::spec::PackagePointerSpec;
-use crate::config::{not_found, Config, ConfigSpec, ConfigureContext};
-use crate::context::RpcContext;
-use crate::db::model::{CurrentDependencies, Database};
-use crate::prelude::*;
-use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
-use crate::s9pk::manifest::{Manifest, PackageId};
-use crate::status::DependencyConfigErrors;
-use crate::util::serde::display_serializable;
-use crate::util::{display_none, Version};
-use crate::volume::Volumes;
-use crate::Error;
-
-#[command(subcommands(configure))]
-pub fn dependency() -> Result<(), Error> {
-    Ok(())
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
-#[model = "Model<Self>"]
-pub struct Dependencies(pub BTreeMap<PackageId, DepInfo>);
-impl Map for Dependencies {
-    type Key = PackageId;
-    type Value = DepInfo;
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-#[serde(tag = "type")]
-pub enum DependencyRequirement {
-    OptIn { how: String },
-    OptOut { how: String },
-    Required,
-}
-impl DependencyRequirement {
-    pub fn required(&self) -> bool {
-        matches!(self, &DependencyRequirement::Required)
-    }
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct DepInfo {
-    pub version: VersionRange,
-    pub requirement: DependencyRequirement,
-    pub description: Option<String>,
-    #[serde(default)]
-    pub config: Option<DependencyConfig>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct DependencyConfig {
-    check: PackageProcedure,
-    auto_configure: PackageProcedure,
-}
-impl DependencyConfig {
-    pub async fn check(
-        &self,
-        ctx: &RpcContext,
-        dependent_id: &PackageId,
-        dependent_version: &Version,
-        dependent_volumes: &Volumes,
-        dependency_id: &PackageId,
-        dependency_config: &Config,
-    ) -> Result<Result<NoOutput, String>, Error> {
-        Ok(self
-            .check
-            .sandboxed(
-                ctx,
-                dependent_id,
-                dependent_version,
-                dependent_volumes,
-                Some(dependency_config),
-                None,
-                ProcedureName::Check(dependency_id.clone()),
-            )
-            .await?
-            .map_err(|(_, e)| e))
-    }
-    pub async fn auto_configure(
-        &self,
-        ctx: &RpcContext,
-        dependent_id: &PackageId,
-        dependent_version: &Version,
-        dependent_volumes: &Volumes,
-        old: &Config,
-    ) -> Result<Config, Error> {
-        self.auto_configure
-            .sandboxed(
-                ctx,
-                dependent_id,
-                dependent_version,
-                dependent_volumes,
-                Some(old),
-                None,
-                ProcedureName::AutoConfig(dependent_id.clone()),
-            )
-            .await?
-            .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))
-    }
-}
-
-#[command(
-    subcommands(self(configure_impl(async)), configure_dry),
-    display(display_none)
-)]
-pub async fn configure(
-    #[arg(rename = "dependent-id")] dependent_id: PackageId,
-    #[arg(rename = "dependency-id")] dependency_id: PackageId,
-) -> Result<(PackageId, PackageId), Error> {
-    Ok((dependent_id, dependency_id))
-}
-
-pub async fn configure_impl(
-    ctx: RpcContext,
-    (pkg_id, dep_id): (PackageId, PackageId),
-) -> Result<(), Error> {
-    let breakages = BTreeMap::new();
-    let overrides = Default::default();
-    let ConfigDryRes {
-        old_config: _,
-        new_config,
-        spec: _,
-    } = configure_logic(ctx.clone(), (pkg_id, dep_id.clone())).await?;
-
-    let configure_context = ConfigureContext {
-        breakages,
-        timeout: Some(Duration::from_secs(3).into()),
-        config: Some(new_config),
-        dry_run: false,
-        overrides,
-    };
-    crate::config::configure(&ctx, &dep_id, configure_context).await?;
-    Ok(())
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct ConfigDryRes {
-    pub old_config: Config,
-    pub new_config: Config,
-    pub spec: ConfigSpec,
-}
-
-#[command(rename = "dry", display(display_serializable))]
-#[instrument(skip_all)]
-pub async fn configure_dry(
-    #[context] ctx: RpcContext,
-    #[parent_data] (pkg_id, dependency_id): (PackageId, PackageId),
-) -> Result<ConfigDryRes, Error> {
-    configure_logic(ctx, (pkg_id, dependency_id)).await
-}
-
-pub async fn configure_logic(
-    ctx: RpcContext,
-    (pkg_id, dependency_id): (PackageId, PackageId),
-) -> Result<ConfigDryRes, Error> {
-    let db = ctx.db.peek().await;
-    let pkg = db
-        .as_package_data()
-        .as_idx(&pkg_id)
-        .or_not_found(&pkg_id)?
-        .as_installed()
-        .or_not_found(&pkg_id)?;
-    let pkg_version = pkg.as_manifest().as_version().de()?;
-    let pkg_volumes = pkg.as_manifest().as_volumes().de()?;
-    let dependency = db
-        .as_package_data()
-        .as_idx(&dependency_id)
-        .or_not_found(&dependency_id)?
-        .as_installed()
-        .or_not_found(&dependency_id)?;
-    let dependency_config_action = dependency
-        .as_manifest()
-        .as_config()
-        .de()?
-        .ok_or_else(|| not_found!("Manifest Config"))?;
-    let dependency_version = dependency.as_manifest().as_version().de()?;
-    let dependency_volumes = dependency.as_manifest().as_volumes().de()?;
-    let dependency = pkg
-        .as_manifest()
-        .as_dependencies()
-        .as_idx(&dependency_id)
-        .or_not_found(&dependency_id)?;
-
-    let ConfigRes {
-        config: maybe_config,
-        spec,
-    } = dependency_config_action
-        .get(
-            &ctx,
-            &dependency_id,
-            &dependency_version,
-            &dependency_volumes,
-        )
-        .await?;
-
-    let old_config = if let Some(config) = maybe_config {
-        config
-    } else {
-        spec.gen(
-            &mut rand::rngs::StdRng::from_entropy(),
-            &Some(Duration::new(10, 0)),
-        )?
-    };
-
-    let new_config = dependency
-        .as_config()
-        .de()?
-        .ok_or_else(|| not_found!("Config"))?
-        .auto_configure
-        .sandboxed(
-            &ctx,
-            &pkg_id,
-            &pkg_version,
-            &pkg_volumes,
-            Some(&old_config),
-            None,
-            ProcedureName::AutoConfig(dependency_id.clone()),
-        )
-        .await?
-        .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))?;
-
-    Ok(ConfigDryRes {
-        old_config,
-        new_config,
-        spec,
-    })
-}
-
-#[instrument(skip_all)]
-pub fn add_dependent_to_current_dependents_lists(
-    db: &mut Model<Database>,
-    dependent_id: &PackageId,
-    current_dependencies: &CurrentDependencies,
-) -> Result<(), Error> {
-    for (dependency, dep_info) in &current_dependencies.0 {
-        if let Some(dependency_dependents) = db
-            .as_package_data_mut()
-            .as_idx_mut(dependency)
-            .and_then(|pde| pde.as_installed_mut())
-            .map(|i| i.as_current_dependents_mut())
-        {
-            dependency_dependents.insert(dependent_id, dep_info)?;
-        }
-    }
-    Ok(())
-}
-
-pub fn set_dependents_with_live_pointers_to_needs_config(
-    db: &mut Peeked,
-    id: &PackageId,
-) -> Result<Vec<(PackageId, Version)>, Error> {
-    let mut res = Vec::new();
-    for (dep, info) in db
-        .as_package_data()
-        .as_idx(id)
-        .or_not_found(id)?
-        .as_installed()
-        .or_not_found(id)?
-        .as_current_dependents()
-        .de()?
-        .0
-    {
-        if info.pointers.iter().any(|ptr| match ptr {
-            // dependency id matches the package being uninstalled
-            PackagePointerSpec::TorAddress(ptr) => &ptr.package_id == id && &dep != id,
-            PackagePointerSpec::LanAddress(ptr) => &ptr.package_id == id && &dep != id,
-            // we never need to retarget these
-            PackagePointerSpec::TorKey(_) => false,
-            PackagePointerSpec::Config(_) => false,
-        }) {
-            let installed = db
-                .as_package_data_mut()
-                .as_idx_mut(&dep)
-                .or_not_found(&dep)?
-                .as_installed_mut()
-                .or_not_found(&dep)?;
-            let version = installed.as_manifest().as_version().de()?;
-            let configured = installed.as_status_mut().as_configured_mut();
-            if configured.de()? {
-                configured.ser(&false)?;
-                res.push((dep, version));
-            }
-        }
-    }
-    Ok(res)
-}
-
-#[instrument(skip_all)]
-pub async fn compute_dependency_config_errs(
-    ctx: &RpcContext,
-    db: &Peeked,
-    manifest: &Manifest,
-    current_dependencies: &CurrentDependencies,
-    dependency_config: &BTreeMap<PackageId, Config>,
-) -> Result<DependencyConfigErrors, Error> {
-    let mut dependency_config_errs = BTreeMap::new();
-    for (dependency, _dep_info) in current_dependencies
-        .0
-        .iter()
-        .filter(|(dep_id, _)| dep_id != &&manifest.id)
-    {
-        // check if config passes dependency check
-        if let Some(cfg) = &manifest
-            .dependencies
-            .0
-            .get(dependency)
-            .or_not_found(dependency)?
-            .config
-        {
-            if let Err(error) = cfg
-                .check(
-                    ctx,
-                    &manifest.id,
-                    &manifest.version,
-                    &manifest.volumes,
-                    dependency,
-                    &if let Some(config) = dependency_config.get(dependency) {
-                        config.clone()
-                    } else if let Some(manifest) = db
-                        .as_package_data()
-                        .as_idx(dependency)
-                        .and_then(|pde| pde.as_installed())
-                        .map(|i| i.as_manifest().de())
-                        .transpose()?
-                    {
-                        if let Some(config) = &manifest.config {
-                            config
-                                .get(ctx, &manifest.id, &manifest.version, &manifest.volumes)
-                                .await?
-                                .config
-                                .unwrap_or_default()
-                        } else {
-                            Config::default()
-                        }
-                    } else {
-                        Config::default()
-                    },
-                )
-                .await?
-            {
-                dependency_config_errs.insert(dependency.clone(), error);
-            }
-        }
-    }
-    Ok(DependencyConfigErrors(dependency_config_errs))
-}

backend/src/diagnostic.rs

@@ -1,72 +0,0 @@
-use std::path::Path;
-use std::sync::Arc;
-
-use rpc_toolkit::command;
-use rpc_toolkit::yajrc::RpcError;
-
-use crate::context::DiagnosticContext;
-use crate::disk::repair;
-use crate::init::SYSTEM_REBUILD_PATH;
-use crate::logs::{fetch_logs, LogResponse, LogSource};
-use crate::shutdown::Shutdown;
-use crate::util::display_none;
-use crate::Error;
-
-#[command(subcommands(error, logs, exit, restart, forget_disk, disk, rebuild))]
-pub fn diagnostic() -> Result<(), Error> {
-    Ok(())
-}
-
-#[command]
-pub fn error(#[context] ctx: DiagnosticContext) -> Result<Arc<RpcError>, Error> {
-    Ok(ctx.error.clone())
-}
-
-#[command(rpc_only)]
-pub async fn logs(
-    #[arg] limit: Option<usize>,
-    #[arg] cursor: Option<String>,
-    #[arg] before: bool,
-) -> Result<LogResponse, Error> {
-    Ok(fetch_logs(LogSource::System, limit, cursor, before).await?)
-}
-
-#[command(display(display_none))]
-pub fn exit(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
-    ctx.shutdown.send(None).expect("receiver dropped");
-    Ok(())
-}
-
-#[command(display(display_none))]
-pub fn restart(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
-    ctx.shutdown
-        .send(Some(Shutdown {
-            export_args: ctx
-                .disk_guid
-                .clone()
-                .map(|guid| (guid, ctx.datadir.clone())),
-            restart: true,
-        }))
-        .expect("receiver dropped");
-    Ok(())
-}
-
-#[command(display(display_none))]
-pub async fn rebuild(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
-    tokio::fs::write(SYSTEM_REBUILD_PATH, b"").await?;
-    restart(ctx)
-}
-
-#[command(subcommands(forget_disk, repair))]
-pub fn disk() -> Result<(), Error> {
-    Ok(())
-}
-
-#[command(rename = "forget", display(display_none))]
-pub async fn forget_disk() -> Result<(), Error> {
-    let disk_guid = Path::new("/media/embassy/config/disk.guid");
-    if tokio::fs::metadata(disk_guid).await.is_ok() {
-        tokio::fs::remove_file(disk_guid).await?;
-    }
-    Ok(())
-}

backend/src/disk/mount/filesystem/ecryptfs.rs

@@ -1,71 +0,0 @@
-use std::os::unix::ffi::OsStrExt;
-use std::path::Path;
-
-use async_trait::async_trait;
-use digest::generic_array::GenericArray;
-use digest::{Digest, OutputSizeUser};
-use sha2::Sha256;
-
-use super::{FileSystem, MountType};
-use crate::util::Invoke;
-use crate::{Error, ResultExt};
-
-pub async fn mount_ecryptfs<P0: AsRef<Path>, P1: AsRef<Path>>(
-    src: P0,
-    dst: P1,
-    key: &str,
-) -> Result<(), Error> {
-    tokio::fs::create_dir_all(dst.as_ref()).await?;
-    tokio::process::Command::new("mount")
-        .arg("-t")
-        .arg("ecryptfs")
-        .arg(src.as_ref())
-        .arg(dst.as_ref())
-        .arg("-o")
-        // for more information `man ecryptfs` 
-        .arg(format!("key=passphrase:passphrase_passwd={},ecryptfs_cipher=aes,ecryptfs_key_bytes=32,ecryptfs_passthrough=n,ecryptfs_enable_filename_crypto=y,no_sig_cache", key))
-        .input(Some(&mut std::io::Cursor::new(b"\n")))
-        .invoke(crate::ErrorKind::Filesystem).await?;
-    Ok(())
-}
-
-pub struct EcryptFS<EncryptedDir: AsRef<Path>, Key: AsRef<str>> {
-    encrypted_dir: EncryptedDir,
-    key: Key,
-}
-impl<EncryptedDir: AsRef<Path>, Key: AsRef<str>> EcryptFS<EncryptedDir, Key> {
-    pub fn new(encrypted_dir: EncryptedDir, key: Key) -> Self {
-        EcryptFS { encrypted_dir, key }
-    }
-}
-#[async_trait]
-impl<EncryptedDir: AsRef<Path> + Send + Sync, Key: AsRef<str> + Send + Sync> FileSystem
-    for EcryptFS<EncryptedDir, Key>
-{
-    async fn mount<P: AsRef<Path> + Send + Sync>(
-        &self,
-        mountpoint: P,
-        _mount_type: MountType, // ignored - inherited from parent fs
-    ) -> Result<(), Error> {
-        mount_ecryptfs(self.encrypted_dir.as_ref(), mountpoint, self.key.as_ref()).await
-    }
-    async fn source_hash(
-        &self,
-    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
-        let mut sha = Sha256::new();
-        sha.update("EcryptFS");
-        sha.update(
-            tokio::fs::canonicalize(self.encrypted_dir.as_ref())
-                .await
-                .with_ctx(|_| {
-                    (
-                        crate::ErrorKind::Filesystem,
-                        self.encrypted_dir.as_ref().display().to_string(),
-                    )
-                })?
-                .as_os_str()
-                .as_bytes(),
-        );
-        Ok(sha.finalize())
-    }
-}

backend/src/disk/mount/filesystem/efivarfs.rs

@@ -1,39 +0,0 @@
-use std::path::Path;
-
-use async_trait::async_trait;
-use digest::generic_array::GenericArray;
-use digest::{Digest, OutputSizeUser};
-use sha2::Sha256;
-
-use super::{FileSystem, MountType, ReadOnly};
-use crate::util::Invoke;
-use crate::Error;
-
-pub struct EfiVarFs;
-#[async_trait]
-impl FileSystem for EfiVarFs {
-    async fn mount<P: AsRef<Path> + Send + Sync>(
-        &self,
-        mountpoint: P,
-        mount_type: MountType,
-    ) -> Result<(), Error> {
-        tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
-        let mut cmd = tokio::process::Command::new("mount");
-        cmd.arg("-t")
-            .arg("efivarfs")
-            .arg("efivarfs")
-            .arg(mountpoint.as_ref());
-        if mount_type == ReadOnly {
-            cmd.arg("-o").arg("ro");
-        }
-        cmd.invoke(crate::ErrorKind::Filesystem).await?;
-        Ok(())
-    }
-    async fn source_hash(
-        &self,
-    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
-        let mut sha = Sha256::new();
-        sha.update("EfiVarFs");
-        Ok(sha.finalize())
-    }
-}

backend/src/disk/mount/filesystem/label.rs

@@ -1,52 +0,0 @@
-use std::path::Path;
-
-use async_trait::async_trait;
-use digest::generic_array::GenericArray;
-use digest::{Digest, OutputSizeUser};
-use sha2::Sha256;
-
-use super::{FileSystem, MountType, ReadOnly};
-use crate::util::Invoke;
-use crate::Error;
-
-pub async fn mount_label(
-    label: &str,
-    mountpoint: impl AsRef<Path>,
-    mount_type: MountType,
-) -> Result<(), Error> {
-    tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
-    let mut cmd = tokio::process::Command::new("mount");
-    cmd.arg("-L").arg(label).arg(mountpoint.as_ref());
-    if mount_type == ReadOnly {
-        cmd.arg("-o").arg("ro");
-    }
-    cmd.invoke(crate::ErrorKind::Filesystem).await?;
-    Ok(())
-}
-
-pub struct Label<S: AsRef<str>> {
-    label: S,
-}
-impl<S: AsRef<str>> Label<S> {
-    pub fn new(label: S) -> Self {
-        Label { label }
-    }
-}
-#[async_trait]
-impl<S: AsRef<str> + Send + Sync> FileSystem for Label<S> {
-    async fn mount<P: AsRef<Path> + Send + Sync>(
-        &self,
-        mountpoint: P,
-        mount_type: MountType,
-    ) -> Result<(), Error> {
-        mount_label(self.label.as_ref(), mountpoint, mount_type).await
-    }
-    async fn source_hash(
-        &self,
-    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
-        let mut sha = Sha256::new();
-        sha.update("Label");
-        sha.update(self.label.as_ref().as_bytes());
-        Ok(sha.finalize())
-    }
-}

backend/src/disk/mount/filesystem/mod.rs

@@ -1,36 +0,0 @@
-use std::path::Path;
-
-use async_trait::async_trait;
-use digest::generic_array::GenericArray;
-use digest::OutputSizeUser;
-use sha2::Sha256;
-
-use crate::Error;
-
-pub mod bind;
-pub mod block_dev;
-pub mod cifs;
-pub mod ecryptfs;
-pub mod efivarfs;
-pub mod httpdirfs;
-pub mod label;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum MountType {
-    ReadOnly,
-    ReadWrite,
-}
-
-pub use MountType::*;
-
-#[async_trait]
-pub trait FileSystem {
-    async fn mount<P: AsRef<Path> + Send + Sync>(
-        &self,
-        mountpoint: P,
-        mount_type: MountType,
-    ) -> Result<(), Error>;
-    async fn source_hash(
-        &self,
-    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error>;
-}

backend/src/firmware.rs

@@ -1,70 +0,0 @@
-use std::path::Path;
-
-use async_compression::tokio::bufread::GzipDecoder;
-use tokio::fs::File;
-use tokio::io::{AsyncRead, BufReader};
-use tokio::process::Command;
-
-use crate::disk::fsck::RequiresReboot;
-use crate::prelude::*;
-use crate::util::Invoke;
-
-pub async fn update_firmware() -> Result<RequiresReboot, Error> {
-    let product_name = String::from_utf8(
-        Command::new("dmidecode")
-            .arg("-s")
-            .arg("system-product-name")
-            .invoke(ErrorKind::Firmware)
-            .await?,
-    )?
-    .trim()
-    .to_owned();
-    if product_name.is_empty() {
-        return Ok(RequiresReboot(false));
-    }
-    let firmware_dir = Path::new("/usr/lib/startos/firmware").join(&product_name);
-    if tokio::fs::metadata(&firmware_dir).await.is_ok() {
-        let current_firmware = String::from_utf8(
-            Command::new("dmidecode")
-                .arg("-s")
-                .arg("bios-version")
-                .invoke(ErrorKind::Firmware)
-                .await?,
-        )?
-        .trim()
-        .to_owned();
-        if tokio::fs::metadata(firmware_dir.join(format!("{current_firmware}.rom.gz")))
-            .await
-            .is_err()
-            && tokio::fs::metadata(firmware_dir.join(format!("{current_firmware}.rom")))
-                .await
-                .is_err()
-        {
-            let mut firmware_read_dir = tokio::fs::read_dir(&firmware_dir).await?;
-            while let Some(entry) = firmware_read_dir.next_entry().await? {
-                let filename = entry.file_name().to_string_lossy().into_owned();
-                let rdr: Option<Box<dyn AsyncRead + Unpin + Send>> =
-                    if filename.ends_with(".rom.gz") {
-                        Some(Box::new(GzipDecoder::new(BufReader::new(
-                            File::open(entry.path()).await?,
-                        ))))
-                    } else if filename.ends_with(".rom") {
-                        Some(Box::new(File::open(entry.path()).await?))
-                    } else {
-                        None
-                    };
-                if let Some(mut rdr) = rdr {
-                    Command::new("flashrom")
-                        .arg("-p")
-                        .arg("internal")
-                        .arg("-w-")
-                        .input(Some(&mut rdr))
-                        .invoke(ErrorKind::Firmware)
-                        .await?;
-                    return Ok(RequiresReboot(true));
-                }
-            }
-        }
-    }
-    Ok(RequiresReboot(false))
-}

backend/src/init.rs

@@ -1,455 +0,0 @@
-use std::fs::Permissions;
-use std::os::unix::fs::PermissionsExt;
-use std::path::Path;
-use std::time::{Duration, SystemTime};
-
-use color_eyre::eyre::eyre;
-use helpers::NonDetachingJoinHandle;
-use models::ResultExt;
-use rand::random;
-use sqlx::{Pool, Postgres};
-use tokio::process::Command;
-use tracing::instrument;
-
-use crate::account::AccountInfo;
-use crate::context::rpc::RpcContextConfig;
-use crate::db::model::ServerStatus;
-use crate::disk::mount::util::unmount;
-use crate::install::PKG_ARCHIVE_DIR;
-use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
-use crate::prelude::*;
-use crate::sound::BEP;
-use crate::util::cpupower::{
-    current_governor, get_available_governors, set_governor, GOVERNOR_PERFORMANCE,
-};
-use crate::util::docker::{create_bridge_network, CONTAINER_DATADIR, CONTAINER_TOOL};
-use crate::util::Invoke;
-use crate::{Error, ARCH};
-
-pub const SYSTEM_REBUILD_PATH: &str = "/media/embassy/config/system-rebuild";
-pub const STANDBY_MODE_PATH: &str = "/media/embassy/config/standby";
-
-pub async fn check_time_is_synchronized() -> Result<bool, Error> {
-    Ok(String::from_utf8(
-        Command::new("timedatectl")
-            .arg("show")
-            .arg("-p")
-            .arg("NTPSynchronized")
-            .invoke(crate::ErrorKind::Unknown)
-            .await?,
-    )?
-    .trim()
-        == "NTPSynchronized=yes")
-}
-
-// must be idempotent
-#[tracing::instrument(skip_all)]
-pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
-    let db_dir = datadir.as_ref().join("main/postgresql");
-    if tokio::process::Command::new("mountpoint")
-        .arg("/var/lib/postgresql")
-        .stdout(std::process::Stdio::null())
-        .stderr(std::process::Stdio::null())
-        .status()
-        .await?
-        .success()
-    {
-        unmount("/var/lib/postgresql").await?;
-    }
-    let exists = tokio::fs::metadata(&db_dir).await.is_ok();
-    if !exists {
-        Command::new("cp")
-            .arg("-ra")
-            .arg("/var/lib/postgresql")
-            .arg(&db_dir)
-            .invoke(crate::ErrorKind::Filesystem)
-            .await?;
-    }
-    Command::new("chown")
-        .arg("-R")
-        .arg("postgres:postgres")
-        .arg(&db_dir)
-        .invoke(crate::ErrorKind::Database)
-        .await?;
-
-    let mut pg_paths = tokio::fs::read_dir("/usr/lib/postgresql").await?;
-    let mut pg_version = None;
-    while let Some(pg_path) = pg_paths.next_entry().await? {
-        let pg_path_version = pg_path
-            .file_name()
-            .to_str()
-            .map(|v| v.parse())
-            .transpose()?
-            .unwrap_or(0);
-        if pg_path_version > pg_version.unwrap_or(0) {
-            pg_version = Some(pg_path_version)
-        }
-    }
-    let pg_version = pg_version.ok_or_else(|| {
-        Error::new(
-            eyre!("could not determine postgresql version"),
-            crate::ErrorKind::Database,
-        )
-    })?;
-
-    crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
-
-    let pg_version_string = pg_version.to_string();
-    let pg_version_path = db_dir.join(&pg_version_string);
-    if tokio::fs::metadata(&pg_version_path).await.is_err() {
-        let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
-        let conf_dir_tmp = {
-            let mut tmp = conf_dir.clone();
-            tmp.set_extension("tmp");
-            tmp
-        };
-        if tokio::fs::metadata(&conf_dir).await.is_ok() {
-            Command::new("mv")
-                .arg(&conf_dir)
-                .arg(&conf_dir_tmp)
-                .invoke(ErrorKind::Filesystem)
-                .await?;
-        }
-        let mut old_version = pg_version;
-        while old_version > 13
-        /* oldest pg version included in startos */
-        {
-            old_version -= 1;
-            let old_datadir = db_dir.join(old_version.to_string());
-            if tokio::fs::metadata(&old_datadir).await.is_ok() {
-                Command::new("pg_upgradecluster")
-                    .arg(old_version.to_string())
-                    .arg("main")
-                    .invoke(crate::ErrorKind::Database)
-                    .await?;
-                break;
-            }
-        }
-        if tokio::fs::metadata(&conf_dir).await.is_ok() {
-            if tokio::fs::metadata(&conf_dir).await.is_ok() {
-                tokio::fs::remove_dir_all(&conf_dir).await?;
-            }
-            Command::new("mv")
-                .arg(&conf_dir_tmp)
-                .arg(&conf_dir)
-                .invoke(ErrorKind::Filesystem)
-                .await?;
-        }
-    }
-
-    Command::new("systemctl")
-        .arg("start")
-        .arg(format!("postgresql@{pg_version}-main.service"))
-        .invoke(crate::ErrorKind::Database)
-        .await?;
-    if !exists {
-        Command::new("sudo")
-            .arg("-u")
-            .arg("postgres")
-            .arg("createuser")
-            .arg("root")
-            .invoke(crate::ErrorKind::Database)
-            .await?;
-        Command::new("sudo")
-            .arg("-u")
-            .arg("postgres")
-            .arg("createdb")
-            .arg("secrets")
-            .arg("-O")
-            .arg("root")
-            .invoke(crate::ErrorKind::Database)
-            .await?;
-    }
-
-    Ok(())
-}
-
-pub struct InitResult {
-    pub secret_store: Pool<Postgres>,
-    pub db: patch_db::PatchDb,
-}
-
-#[instrument(skip_all)]
-pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
-    tokio::fs::create_dir_all("/run/embassy")
-        .await
-        .with_ctx(|_| (crate::ErrorKind::Filesystem, "mkdir -p /run/embassy"))?;
-    if tokio::fs::metadata(LOCAL_AUTH_COOKIE_PATH).await.is_err() {
-        tokio::fs::write(
-            LOCAL_AUTH_COOKIE_PATH,
-            base64::encode(random::<[u8; 32]>()).as_bytes(),
-        )
-        .await
-        .with_ctx(|_| {
-            (
-                crate::ErrorKind::Filesystem,
-                format!("write {}", LOCAL_AUTH_COOKIE_PATH),
-            )
-        })?;
-        tokio::fs::set_permissions(LOCAL_AUTH_COOKIE_PATH, Permissions::from_mode(0o046)).await?;
-        Command::new("chown")
-            .arg("root:embassy")
-            .arg(LOCAL_AUTH_COOKIE_PATH)
-            .invoke(crate::ErrorKind::Filesystem)
-            .await?;
-    }
-
-    let secret_store = cfg.secret_store().await?;
-    tracing::info!("Opened Postgres");
-
-    crate::ssh::sync_keys_from_db(&secret_store, "/home/start9/.ssh/authorized_keys").await?;
-    tracing::info!("Synced SSH Keys");
-
-    let account = AccountInfo::load(&secret_store).await?;
-    let db = cfg.db(&account).await?;
-    tracing::info!("Opened PatchDB");
-    let peek = db.peek().await;
-    let mut server_info = peek.as_server_info().de()?;
-
-    // write to ca cert store
-    tokio::fs::write(
-        "/usr/local/share/ca-certificates/startos-root-ca.crt",
-        account.root_ca_cert.to_pem()?,
-    )
-    .await?;
-    Command::new("update-ca-certificates")
-        .invoke(crate::ErrorKind::OpenSsl)
-        .await?;
-
-    if let Some(wifi_interface) = &cfg.wifi_interface {
-        crate::net::wifi::synchronize_wpa_supplicant_conf(
-            &cfg.datadir().join("main"),
-            wifi_interface,
-            &server_info.last_wifi_region,
-        )
-        .await?;
-        tracing::info!("Synchronized WiFi");
-    }
-
-    let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
-        || &*server_info.version < &emver::Version::new(0, 3, 2, 0)
-        || (*ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
-
-    let song = if should_rebuild {
-        Some(NonDetachingJoinHandle::from(tokio::spawn(async {
-            loop {
-                BEP.play().await.unwrap();
-                BEP.play().await.unwrap();
-                tokio::time::sleep(Duration::from_secs(60)).await;
-            }
-        })))
-    } else {
-        None
-    };
-
-    let log_dir = cfg.datadir().join("main/logs");
-    if tokio::fs::metadata(&log_dir).await.is_err() {
-        tokio::fs::create_dir_all(&log_dir).await?;
-    }
-    let current_machine_id = tokio::fs::read_to_string("/etc/machine-id").await?;
-    let mut machine_ids = tokio::fs::read_dir(&log_dir).await?;
-    while let Some(machine_id) = machine_ids.next_entry().await? {
-        if machine_id.file_name().to_string_lossy().trim() != current_machine_id.trim() {
-            tokio::fs::remove_dir_all(machine_id.path()).await?;
-        }
-    }
-    crate::disk::mount::util::bind(&log_dir, "/var/log/journal", false).await?;
-    match Command::new("chattr")
-        .arg("-R")
-        .arg("+C")
-        .arg("/var/log/journal")
-        .invoke(ErrorKind::Filesystem)
-        .await
-    {
-        Ok(_) => Ok(()),
-        Err(e) if e.source.to_string().contains("Operation not supported") => Ok(()),
-        Err(e) => Err(e),
-    }?;
-    Command::new("systemctl")
-        .arg("restart")
-        .arg("systemd-journald")
-        .invoke(crate::ErrorKind::Journald)
-        .await?;
-    tracing::info!("Mounted Logs");
-
-    let tmp_dir = cfg.datadir().join("package-data/tmp");
-    if should_rebuild && tokio::fs::metadata(&tmp_dir).await.is_ok() {
-        tokio::fs::remove_dir_all(&tmp_dir).await?;
-    }
-    if tokio::fs::metadata(&tmp_dir).await.is_err() {
-        tokio::fs::create_dir_all(&tmp_dir).await?;
-    }
-    let tmp_var = cfg.datadir().join(format!("package-data/tmp/var"));
-    if tokio::fs::metadata(&tmp_var).await.is_ok() {
-        tokio::fs::remove_dir_all(&tmp_var).await?;
-    }
-    crate::disk::mount::util::bind(&tmp_var, "/var/tmp", false).await?;
-    let tmp_docker = cfg
-        .datadir()
-        .join(format!("package-data/tmp/{CONTAINER_TOOL}"));
-    let tmp_docker_exists = tokio::fs::metadata(&tmp_docker).await.is_ok();
-    if CONTAINER_TOOL == "docker" {
-        Command::new("systemctl")
-            .arg("stop")
-            .arg("docker")
-            .invoke(crate::ErrorKind::Docker)
-            .await?;
-    }
-    crate::disk::mount::util::bind(&tmp_docker, CONTAINER_DATADIR, false).await?;
-
-    if CONTAINER_TOOL == "docker" {
-        Command::new("systemctl")
-            .arg("reset-failed")
-            .arg("docker")
-            .invoke(crate::ErrorKind::Docker)
-            .await?;
-        Command::new("systemctl")
-            .arg("start")
-            .arg("docker")
-            .invoke(crate::ErrorKind::Docker)
-            .await?;
-    }
-    tracing::info!("Mounted Docker Data");
-
-    if should_rebuild || !tmp_docker_exists {
-        if CONTAINER_TOOL == "docker" {
-            tracing::info!("Creating Docker Network");
-            create_bridge_network("start9", "172.18.0.1/24", "br-start9").await?;
-            tracing::info!("Created Docker Network");
-        }
-
-        tracing::info!("Loading System Docker Images");
-        crate::install::load_images("/usr/lib/startos/system-images").await?;
-        tracing::info!("Loaded System Docker Images");
-
-        tracing::info!("Loading Package Docker Images");
-        crate::install::load_images(cfg.datadir().join(PKG_ARCHIVE_DIR)).await?;
-        tracing::info!("Loaded Package Docker Images");
-    }
-
-    if CONTAINER_TOOL == "podman" {
-        crate::util::docker::remove_container("netdummy", true).await?;
-        Command::new("podman")
-            .arg("run")
-            .arg("-d")
-            .arg("--rm")
-            .arg("--network=start9")
-            .arg("--name=netdummy")
-            .arg("start9/x_system/utils:latest")
-            .arg("sleep")
-            .arg("infinity")
-            .invoke(crate::ErrorKind::Docker)
-            .await?;
-    }
-
-    tracing::info!("Enabling Docker QEMU Emulation");
-    Command::new(CONTAINER_TOOL)
-        .arg("run")
-        .arg("--privileged")
-        .arg("--rm")
-        .arg("start9/x_system/binfmt")
-        .arg("--install")
-        .arg("all")
-        .invoke(crate::ErrorKind::Docker)
-        .await?;
-    tracing::info!("Enabled Docker QEMU Emulation");
-
-    if current_governor()
-        .await?
-        .map(|g| &g != &GOVERNOR_PERFORMANCE)
-        .unwrap_or(false)
-    {
-        tracing::info!("Setting CPU Governor to \"{}\"", GOVERNOR_PERFORMANCE);
-        if get_available_governors()
-            .await?
-            .contains(&GOVERNOR_PERFORMANCE)
-        {
-            set_governor(&GOVERNOR_PERFORMANCE).await?;
-            tracing::info!("Set CPU Governor");
-        } else {
-            tracing::warn!("CPU Governor \"{}\" Not Available", GOVERNOR_PERFORMANCE)
-        }
-    }
-
-    let mut time_not_synced = true;
-    let mut not_made_progress = 0u32;
-    for _ in 0..1800 {
-        if check_time_is_synchronized().await? {
-            time_not_synced = false;
-            break;
-        }
-        let t = SystemTime::now();
-        tokio::time::sleep(Duration::from_secs(1)).await;
-        if t.elapsed()
-            .map(|t| t > Duration::from_secs_f64(1.1))
-            .unwrap_or(true)
-        {
-            not_made_progress = 0;
-        } else {
-            not_made_progress += 1;
-        }
-        if not_made_progress > 30 {
-            break;
-        }
-    }
-    if time_not_synced {
-        tracing::warn!("Timed out waiting for system time to synchronize");
-    } else {
-        tracing::info!("Syncronized system clock");
-    }
-
-    if server_info.zram {
-        crate::system::enable_zram().await?
-    }
-    server_info.ip_info = crate::net::dhcp::init_ips().await?;
-    server_info.status_info = ServerStatus {
-        updated: false,
-        update_progress: None,
-        backup_progress: None,
-        shutting_down: false,
-        restarting: false,
-    };
-
-    server_info.ntp_synced = if time_not_synced {
-        let db = db.clone();
-        tokio::spawn(async move {
-            while !check_time_is_synchronized().await.unwrap() {
-                tokio::time::sleep(Duration::from_secs(30)).await;
-            }
-            db.mutate(|v| v.as_server_info_mut().as_ntp_synced_mut().ser(&true))
-                .await
-                .unwrap()
-        });
-        false
-    } else {
-        true
-    };
-
-    db.mutate(|v| {
-        v.as_server_info_mut().ser(&server_info)?;
-        Ok(())
-    })
-    .await?;
-
-    crate::version::init(&db, &secret_store).await?;
-
-    db.mutate(|d| {
-        let model = d.de()?;
-        d.ser(&model)
-    })
-    .await?;
-
-    if should_rebuild {
-        match tokio::fs::remove_file(SYSTEM_REBUILD_PATH).await {
-            Ok(()) => Ok(()),
-            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
-            Err(e) => Err(e),
-        }?;
-    }
-
-    drop(song);
-
-    tracing::info!("System initialized.");
-
-    Ok(InitResult { secret_store, db })
-}

backend/src/inspect.rs

@@ -1,92 +0,0 @@
-use std::path::PathBuf;
-
-use rpc_toolkit::command;
-
-use crate::s9pk::manifest::Manifest;
-use crate::s9pk::reader::S9pkReader;
-use crate::util::display_none;
-use crate::util::serde::{display_serializable, IoFormat};
-use crate::Error;
-
-#[command(subcommands(hash, manifest, license, icon, instructions, docker_images))]
-pub fn inspect() -> Result<(), Error> {
-    Ok(())
-}
-
-#[command(cli_only)]
-pub async fn hash(#[arg] path: PathBuf) -> Result<String, Error> {
-    Ok(S9pkReader::open(path, true)
-        .await?
-        .hash_str()
-        .unwrap()
-        .to_owned())
-}
-
-#[command(cli_only, display(display_serializable))]
-pub async fn manifest(
-    #[arg] path: PathBuf,
-    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-) -> Result<Manifest, Error> {
-    S9pkReader::open(path, !no_verify).await?.manifest().await
-}
-
-#[command(cli_only, display(display_none))]
-pub async fn license(
-    #[arg] path: PathBuf,
-    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
-) -> Result<(), Error> {
-    tokio::io::copy(
-        &mut S9pkReader::open(path, !no_verify).await?.license().await?,
-        &mut tokio::io::stdout(),
-    )
-    .await?;
-    Ok(())
-}
-
-#[command(cli_only, display(display_none))]
-pub async fn icon(
-    #[arg] path: PathBuf,
-    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
-) -> Result<(), Error> {
-    tokio::io::copy(
-        &mut S9pkReader::open(path, !no_verify).await?.icon().await?,
-        &mut tokio::io::stdout(),
-    )
-    .await?;
-    Ok(())
-}
-
-#[command(cli_only, display(display_none))]
-pub async fn instructions(
-    #[arg] path: PathBuf,
-    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
-) -> Result<(), Error> {
-    tokio::io::copy(
-        &mut S9pkReader::open(path, !no_verify)
-            .await?
-            .instructions()
-            .await?,
-        &mut tokio::io::stdout(),
-    )
-    .await?;
-    Ok(())
-}
-
-#[command(cli_only, display(display_none), rename = "docker-images")]
-pub async fn docker_images(
-    #[arg] path: PathBuf,
-    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
-) -> Result<(), Error> {
-    tokio::io::copy(
-        &mut S9pkReader::open(path, !no_verify)
-            .await?
-            .docker_images()
-            .await?,
-        &mut tokio::io::stdout(),
-    )
-    .await?;
-    Ok(())
-}

backend/src/install/cleanup.rs

@@ -1,241 +0,0 @@
-use std::path::PathBuf;
-use std::sync::Arc;
-
-use models::OptionExt;
-use sqlx::{Executor, Postgres};
-use tracing::instrument;
-
-use super::PKG_ARCHIVE_DIR;
-use crate::context::RpcContext;
-use crate::db::model::{
-    CurrentDependencies, Database, PackageDataEntry, PackageDataEntryInstalled,
-    PackageDataEntryMatchModelRef,
-};
-use crate::error::ErrorCollection;
-use crate::prelude::*;
-use crate::s9pk::manifest::PackageId;
-use crate::util::{Apply, Version};
-use crate::volume::{asset_dir, script_dir};
-use crate::Error;
-
-#[instrument(skip_all)]
-pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Result<(), Error> {
-    let mut errors = ErrorCollection::new();
-    ctx.managers.remove(&(id.clone(), version.clone())).await;
-    // docker images start9/$APP_ID/*:$VERSION -q | xargs docker rmi
-    let images = crate::util::docker::images_for(id, version).await?;
-    errors.extend(
-        futures::future::join_all(images.into_iter().map(|sha| async {
-            let sha = sha; // move into future
-            crate::util::docker::remove_image(&sha).await
-        }))
-        .await,
-    );
-    let pkg_archive_dir = ctx
-        .datadir
-        .join(PKG_ARCHIVE_DIR)
-        .join(id)
-        .join(version.as_str());
-    if tokio::fs::metadata(&pkg_archive_dir).await.is_ok() {
-        tokio::fs::remove_dir_all(&pkg_archive_dir)
-            .await
-            .apply(|res| errors.handle(res));
-    }
-    let assets_path = asset_dir(&ctx.datadir, id, version);
-    if tokio::fs::metadata(&assets_path).await.is_ok() {
-        tokio::fs::remove_dir_all(&assets_path)
-            .await
-            .apply(|res| errors.handle(res));
-    }
-    let scripts_path = script_dir(&ctx.datadir, id, version);
-    if tokio::fs::metadata(&scripts_path).await.is_ok() {
-        tokio::fs::remove_dir_all(&scripts_path)
-            .await
-            .apply(|res| errors.handle(res));
-    }
-
-    errors.into_result()
-}
-
-#[instrument(skip_all)]
-pub async fn cleanup_failed(ctx: &RpcContext, id: &PackageId) -> Result<(), Error> {
-    if let Some(version) = match ctx
-        .db
-        .peek()
-        .await
-        .as_package_data()
-        .as_idx(id)
-        .or_not_found(id)?
-        .as_match()
-    {
-        PackageDataEntryMatchModelRef::Installing(m) => Some(m.as_manifest().as_version().de()?),
-        PackageDataEntryMatchModelRef::Restoring(m) => Some(m.as_manifest().as_version().de()?),
-        PackageDataEntryMatchModelRef::Updating(m) => {
-            let manifest_version = m.as_manifest().as_version().de()?;
-            let installed = m.as_installed().as_manifest().as_version().de()?;
-            if manifest_version != installed {
-                Some(manifest_version)
-            } else {
-                None // do not remove existing data
-            }
-        }
-        _ => {
-            tracing::warn!("{}: Nothing to clean up!", id);
-            None
-        }
-    } {
-        cleanup(ctx, id, &version).await?;
-    }
-
-    ctx.db
-        .mutate(|v| {
-            match v
-                .clone()
-                .into_package_data()
-                .into_idx(id)
-                .or_not_found(id)?
-                .as_match()
-            {
-                PackageDataEntryMatchModelRef::Installing(_)
-                | PackageDataEntryMatchModelRef::Restoring(_) => {
-                    v.as_package_data_mut().remove(id)?;
-                }
-                PackageDataEntryMatchModelRef::Updating(pde) => {
-                    v.as_package_data_mut()
-                        .as_idx_mut(id)
-                        .or_not_found(id)?
-                        .ser(&PackageDataEntry::Installed(PackageDataEntryInstalled {
-                            manifest: pde.as_installed().as_manifest().de()?,
-                            static_files: pde.as_static_files().de()?,
-                            installed: pde.as_installed().de()?,
-                        }))?;
-                }
-                _ => (),
-            }
-            Ok(())
-        })
-        .await
-}
-
-#[instrument(skip_all)]
-pub fn remove_from_current_dependents_lists(
-    db: &mut Model<Database>,
-    id: &PackageId,
-    current_dependencies: &CurrentDependencies,
-) -> Result<(), Error> {
-    for dep in current_dependencies.0.keys().chain(std::iter::once(id)) {
-        if let Some(current_dependents) = db
-            .as_package_data_mut()
-            .as_idx_mut(dep)
-            .and_then(|d| d.as_installed_mut())
-            .map(|i| i.as_current_dependents_mut())
-        {
-            current_dependents.remove(id)?;
-        }
-    }
-    Ok(())
-}
-
-#[instrument(skip_all)]
-pub async fn uninstall<Ex>(ctx: &RpcContext, secrets: &mut Ex, id: &PackageId) -> Result<(), Error>
-where
-    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-{
-    let db = ctx.db.peek().await;
-    let entry = db
-        .as_package_data()
-        .as_idx(id)
-        .or_not_found(id)?
-        .expect_as_removing()?;
-
-    let dependents_paths: Vec<PathBuf> = entry
-        .as_removing()
-        .as_current_dependents()
-        .keys()?
-        .into_iter()
-        .filter(|x| x != id)
-        .flat_map(|x| db.as_package_data().as_idx(&x))
-        .flat_map(|x| x.as_installed())
-        .flat_map(|x| x.as_manifest().as_volumes().de())
-        .flat_map(|x| x.values().cloned().collect::<Vec<_>>())
-        .flat_map(|x| x.pointer_path(&ctx.datadir))
-        .collect();
-
-    let volume_dir = ctx
-        .datadir
-        .join(crate::volume::PKG_VOLUME_DIR)
-        .join(&*entry.as_manifest().as_id().de()?);
-    let version = entry.as_removing().as_manifest().as_version().de()?;
-    tracing::debug!(
-        "Cleaning up {:?} except for {:?}",
-        volume_dir,
-        dependents_paths
-    );
-    cleanup(ctx, id, &version).await?;
-    cleanup_folder(volume_dir, Arc::new(dependents_paths)).await;
-    remove_network_keys(secrets, id).await?;
-
-    ctx.db
-        .mutate(|d| {
-            d.as_package_data_mut().remove(id)?;
-            remove_from_current_dependents_lists(
-                d,
-                id,
-                &entry.as_removing().as_current_dependencies().de()?,
-            )
-        })
-        .await
-}
-
-#[instrument(skip_all)]
-pub async fn remove_network_keys<Ex>(secrets: &mut Ex, id: &PackageId) -> Result<(), Error>
-where
-    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-{
-    sqlx::query!("DELETE FROM network_keys WHERE package = $1", &*id)
-        .execute(&mut *secrets)
-        .await?;
-    sqlx::query!("DELETE FROM tor WHERE package = $1", &*id)
-        .execute(&mut *secrets)
-        .await?;
-    Ok(())
-}
-
-/// Needed to remove, without removing the folders that are mounted in the other docker containers
-pub fn cleanup_folder(
-    path: PathBuf,
-    dependents_volumes: Arc<Vec<PathBuf>>,
-) -> futures::future::BoxFuture<'static, ()> {
-    Box::pin(async move {
-        let meta_data = match tokio::fs::metadata(&path).await {
-            Ok(a) => a,
-            Err(_e) => {
-                return;
-            }
-        };
-        if !meta_data.is_dir() {
-            tracing::error!("is_not dir, remove {:?}", path);
-            let _ = tokio::fs::remove_file(&path).await;
-            return;
-        }
-        if !dependents_volumes
-            .iter()
-            .any(|v| v.starts_with(&path) || v == &path)
-        {
-            tracing::error!("No parents, remove {:?}", path);
-            let _ = tokio::fs::remove_dir_all(&path).await;
-            return;
-        }
-        let mut read_dir = match tokio::fs::read_dir(&path).await {
-            Ok(a) => a,
-            Err(_e) => {
-                return;
-            }
-        };
-        tracing::error!("Parents, recurse {:?}", path);
-        while let Some(entry) = read_dir.next_entry().await.ok().flatten() {
-            let entry_path = entry.path();
-            cleanup_folder(entry_path, dependents_volumes.clone()).await;
-        }
-    })
-}

backend/src/install/progress.rs

@@ -1,228 +0,0 @@
-use std::future::Future;
-use std::io::SeekFrom;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::sync::Arc;
-use std::task::{Context, Poll};
-use std::time::Duration;
-
-use models::{OptionExt, PackageId};
-use serde::{Deserialize, Serialize};
-use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite};
-
-use crate::db::model::Database;
-use crate::prelude::*;
-
-#[derive(Debug, Deserialize, Serialize, HasModel, Default)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct InstallProgress {
-    pub size: Option<u64>,
-    pub downloaded: AtomicU64,
-    pub download_complete: AtomicBool,
-    pub validated: AtomicU64,
-    pub validation_complete: AtomicBool,
-    pub unpacked: AtomicU64,
-    pub unpack_complete: AtomicBool,
-}
-impl InstallProgress {
-    pub fn new(size: Option<u64>) -> Self {
-        InstallProgress {
-            size,
-            downloaded: AtomicU64::new(0),
-            download_complete: AtomicBool::new(false),
-            validated: AtomicU64::new(0),
-            validation_complete: AtomicBool::new(false),
-            unpacked: AtomicU64::new(0),
-            unpack_complete: AtomicBool::new(false),
-        }
-    }
-    pub fn download_complete(&self) {
-        self.download_complete.store(true, Ordering::SeqCst)
-    }
-    pub async fn track_download(self: Arc<Self>, db: PatchDb, id: PackageId) -> Result<(), Error> {
-        let update = |d: &mut Model<Database>| {
-            d.as_package_data_mut()
-                .as_idx_mut(&id)
-                .or_not_found(&id)?
-                .as_install_progress_mut()
-                .or_not_found("install-progress")?
-                .ser(&self)
-        };
-        while !self.download_complete.load(Ordering::SeqCst) {
-            db.mutate(&update).await?;
-            tokio::time::sleep(Duration::from_millis(300)).await;
-        }
-        db.mutate(&update).await
-    }
-    pub async fn track_download_during<
-        F: FnOnce() -> Fut,
-        Fut: Future<Output = Result<T, Error>>,
-        T,
-    >(
-        self: &Arc<Self>,
-        db: PatchDb,
-        id: &PackageId,
-        f: F,
-    ) -> Result<T, Error> {
-        let tracker = tokio::spawn(self.clone().track_download(db.clone(), id.clone()));
-        let res = f().await;
-        self.download_complete.store(true, Ordering::SeqCst);
-        tracker.await.unwrap()?;
-        res
-    }
-    pub async fn track_read(
-        self: Arc<Self>,
-        db: PatchDb,
-        id: PackageId,
-        complete: Arc<AtomicBool>,
-    ) -> Result<(), Error> {
-        let update = |d: &mut Model<Database>| {
-            d.as_package_data_mut()
-                .as_idx_mut(&id)
-                .or_not_found(&id)?
-                .as_install_progress_mut()
-                .or_not_found("install-progress")?
-                .ser(&self)
-        };
-        while !complete.load(Ordering::SeqCst) {
-            db.mutate(&update).await?;
-            tokio::time::sleep(Duration::from_millis(300)).await;
-        }
-        db.mutate(&update).await
-    }
-    pub async fn track_read_during<
-        F: FnOnce() -> Fut,
-        Fut: Future<Output = Result<T, Error>>,
-        T,
-    >(
-        self: &Arc<Self>,
-        db: PatchDb,
-        id: &PackageId,
-        f: F,
-    ) -> Result<T, Error> {
-        let complete = Arc::new(AtomicBool::new(false));
-        let tracker = tokio::spawn(self.clone().track_read(
-            db.clone(),
-            id.clone(),
-            complete.clone(),
-        ));
-        let res = f().await;
-        complete.store(true, Ordering::SeqCst);
-        tracker.await.unwrap()?;
-        res
-    }
-}
-
-#[pin_project::pin_project]
-#[derive(Debug)]
-pub struct InstallProgressTracker<RW> {
-    #[pin]
-    inner: RW,
-    validating: bool,
-    progress: Arc<InstallProgress>,
-}
-impl<RW> InstallProgressTracker<RW> {
-    pub fn new(inner: RW, progress: Arc<InstallProgress>) -> Self {
-        InstallProgressTracker {
-            inner,
-            validating: true,
-            progress,
-        }
-    }
-    pub fn validated(&mut self) {
-        self.progress
-            .validation_complete
-            .store(true, Ordering::SeqCst);
-        self.validating = false;
-    }
-}
-impl<W: AsyncWrite> AsyncWrite for InstallProgressTracker<W> {
-    fn poll_write(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &[u8],
-    ) -> Poll<Result<usize, std::io::Error>> {
-        let this = self.project();
-        match this.inner.poll_write(cx, buf) {
-            Poll::Ready(Ok(n)) => {
-                this.progress
-                    .downloaded
-                    .fetch_add(n as u64, Ordering::SeqCst);
-                Poll::Ready(Ok(n))
-            }
-            a => a,
-        }
-    }
-    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), std::io::Error>> {
-        let this = self.project();
-        this.inner.poll_flush(cx)
-    }
-    fn poll_shutdown(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-    ) -> Poll<Result<(), std::io::Error>> {
-        let this = self.project();
-        this.inner.poll_shutdown(cx)
-    }
-    fn poll_write_vectored(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        bufs: &[std::io::IoSlice<'_>],
-    ) -> Poll<Result<usize, std::io::Error>> {
-        let this = self.project();
-        match this.inner.poll_write_vectored(cx, bufs) {
-            Poll::Ready(Ok(n)) => {
-                this.progress
-                    .downloaded
-                    .fetch_add(n as u64, Ordering::SeqCst);
-                Poll::Ready(Ok(n))
-            }
-            a => a,
-        }
-    }
-}
-impl<R: AsyncRead> AsyncRead for InstallProgressTracker<R> {
-    fn poll_read(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &mut tokio::io::ReadBuf<'_>,
-    ) -> Poll<std::io::Result<()>> {
-        let this = self.project();
-        let prev = buf.filled().len() as u64;
-        match this.inner.poll_read(cx, buf) {
-            Poll::Ready(Ok(())) => {
-                if *this.validating {
-                    &this.progress.validated
-                } else {
-                    &this.progress.unpacked
-                }
-                .fetch_add(buf.filled().len() as u64 - prev, Ordering::SeqCst);
-
-                Poll::Ready(Ok(()))
-            }
-            a => a,
-        }
-    }
-}
-impl<R: AsyncSeek> AsyncSeek for InstallProgressTracker<R> {
-    fn start_seek(self: Pin<&mut Self>, position: SeekFrom) -> std::io::Result<()> {
-        let this = self.project();
-        this.inner.start_seek(position)
-    }
-    fn poll_complete(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<u64>> {
-        let this = self.project();
-        match this.inner.poll_complete(cx) {
-            Poll::Ready(Ok(n)) => {
-                if *this.validating {
-                    &this.progress.validated
-                } else {
-                    &this.progress.unpacked
-                }
-                .store(n, Ordering::SeqCst);
-                Poll::Ready(Ok(n))
-            }
-            a => a,
-        }
-    }
-}

backend/src/lib.rs

@@ -1,159 +0,0 @@
-#![recursion_limit = "256"]
-
-pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com";
-// pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com";
-pub const BUFFER_SIZE: usize = 1024;
-pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
-pub const TARGET: &str = current_platform::CURRENT_PLATFORM;
-lazy_static::lazy_static! {
-    pub static ref ARCH: &'static str = {
-        let (arch, _) = TARGET.split_once("-").unwrap();
-        arch
-    };
-    pub static ref PLATFORM: String = {
-        if let Ok(platform) = std::fs::read_to_string("/usr/lib/startos/PLATFORM.txt") {
-            platform
-        } else {
-            ARCH.to_string()
-        }
-    };
-    pub static ref SOURCE_DATE: SystemTime = {
-        std::fs::metadata(std::env::current_exe().unwrap()).unwrap().modified().unwrap()
-    };
-}
-
-pub mod account;
-pub mod action;
-pub mod auth;
-pub mod backup;
-pub mod bins;
-pub mod config;
-pub mod context;
-pub mod control;
-pub mod core;
-pub mod db;
-pub mod dependencies;
-pub mod developer;
-pub mod diagnostic;
-pub mod disk;
-pub mod error;
-pub mod firmware;
-pub mod hostname;
-pub mod init;
-pub mod inspect;
-pub mod install;
-pub mod logs;
-pub mod manager;
-pub mod middleware;
-pub mod migration;
-pub mod net;
-pub mod notifications;
-pub mod os_install;
-pub mod prelude;
-pub mod procedure;
-pub mod properties;
-pub mod registry;
-pub mod s9pk;
-pub mod setup;
-pub mod shutdown;
-pub mod sound;
-pub mod ssh;
-pub mod status;
-pub mod system;
-pub mod update;
-pub mod util;
-pub mod version;
-pub mod volume;
-
-use std::time::SystemTime;
-
-pub use config::Config;
-pub use error::{Error, ErrorKind, ResultExt};
-use rpc_toolkit::command;
-use rpc_toolkit::yajrc::RpcError;
-
-#[command(metadata(authenticated = false))]
-pub fn echo(#[arg] message: String) -> Result<String, RpcError> {
-    Ok(message)
-}
-
-#[command(subcommands(
-    version::git_info,
-    echo,
-    inspect::inspect,
-    server,
-    package,
-    net::net,
-    auth::auth,
-    db::db,
-    ssh::ssh,
-    net::wifi::wifi,
-    disk::disk,
-    notifications::notification,
-    backup::backup,
-    registry::marketplace::marketplace,
-))]
-pub fn main_api() -> Result<(), RpcError> {
-    Ok(())
-}
-
-#[command(subcommands(
-    system::time,
-    system::experimental,
-    system::logs,
-    system::kernel_logs,
-    system::metrics,
-    shutdown::shutdown,
-    shutdown::restart,
-    shutdown::rebuild,
-    update::update_system,
-))]
-pub fn server() -> Result<(), RpcError> {
-    Ok(())
-}
-
-#[command(subcommands(
-    action::action,
-    install::install,
-    install::sideload,
-    install::uninstall,
-    install::list,
-    config::config,
-    control::start,
-    control::stop,
-    control::restart,
-    logs::logs,
-    properties::properties,
-    dependencies::dependency,
-    backup::package_backup,
-))]
-pub fn package() -> Result<(), RpcError> {
-    Ok(())
-}
-
-#[command(subcommands(
-    version::git_info,
-    s9pk::pack,
-    developer::verify,
-    developer::init,
-    inspect::inspect,
-    registry::admin::publish,
-))]
-pub fn portable_api() -> Result<(), RpcError> {
-    Ok(())
-}
-
-#[command(subcommands(version::git_info, echo, diagnostic::diagnostic))]
-pub fn diagnostic_api() -> Result<(), RpcError> {
-    Ok(())
-}
-
-#[command(subcommands(version::git_info, echo, setup::setup))]
-pub fn setup_api() -> Result<(), RpcError> {
-    Ok(())
-}
-
-#[command(subcommands(version::git_info, echo, os_install::install))]
-pub fn install_api() -> Result<(), RpcError> {
-    Ok(())
-}

backend/src/logs.rs

@@ -1,551 +0,0 @@
-use std::future::Future;
-use std::marker::PhantomData;
-use std::ops::{Deref, DerefMut};
-use std::process::Stdio;
-use std::time::{Duration, UNIX_EPOCH};
-
-use chrono::{DateTime, Utc};
-use color_eyre::eyre::eyre;
-use futures::stream::BoxStream;
-use futures::{FutureExt, SinkExt, Stream, StreamExt, TryStreamExt};
-use hyper::upgrade::Upgraded;
-use hyper::Error as HyperError;
-use rpc_toolkit::command;
-use rpc_toolkit::yajrc::RpcError;
-use serde::{Deserialize, Serialize};
-use tokio::io::{AsyncBufReadExt, BufReader};
-use tokio::process::{Child, Command};
-use tokio::task::JoinError;
-use tokio_stream::wrappers::LinesStream;
-use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
-use tokio_tungstenite::tungstenite::protocol::CloseFrame;
-use tokio_tungstenite::tungstenite::Message;
-use tokio_tungstenite::WebSocketStream;
-use tracing::instrument;
-
-use crate::context::{CliContext, RpcContext};
-use crate::core::rpc_continuations::{RequestGuid, RpcContinuation};
-use crate::error::ResultExt;
-use crate::procedure::docker::DockerProcedure;
-use crate::s9pk::manifest::PackageId;
-use crate::util::display_none;
-use crate::util::serde::Reversible;
-use crate::{Error, ErrorKind};
-
-#[pin_project::pin_project]
-pub struct LogStream {
-    _child: Child,
-    #[pin]
-    entries: BoxStream<'static, Result<JournalctlEntry, Error>>,
-}
-impl Deref for LogStream {
-    type Target = BoxStream<'static, Result<JournalctlEntry, Error>>;
-    fn deref(&self) -> &Self::Target {
-        &self.entries
-    }
-}
-impl DerefMut for LogStream {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.entries
-    }
-}
-impl Stream for LogStream {
-    type Item = Result<JournalctlEntry, Error>;
-    fn poll_next(
-        self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Option<Self::Item>> {
-        let this = self.project();
-        Stream::poll_next(this.entries, cx)
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.entries.size_hint()
-    }
-}
-
-#[instrument(skip_all)]
-async fn ws_handler<
-    WSFut: Future<Output = Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
->(
-    first_entry: Option<LogEntry>,
-    mut logs: LogStream,
-    ws_fut: WSFut,
-) -> Result<(), Error> {
-    let mut stream = ws_fut
-        .await
-        .with_kind(crate::ErrorKind::Network)?
-        .with_kind(crate::ErrorKind::Unknown)?;
-
-    if let Some(first_entry) = first_entry {
-        stream
-            .send(Message::Text(
-                serde_json::to_string(&first_entry).with_kind(ErrorKind::Serialization)?,
-            ))
-            .await
-            .with_kind(ErrorKind::Network)?;
-    }
-
-    let mut ws_closed = false;
-    while let Some(entry) = tokio::select! {
-        a = logs.try_next() => Some(a?),
-        a = stream.try_next() => { a.with_kind(crate::ErrorKind::Network)?; ws_closed = true; None }
-    } {
-        if let Some(entry) = entry {
-            let (_, log_entry) = entry.log_entry()?;
-            stream
-                .send(Message::Text(
-                    serde_json::to_string(&log_entry).with_kind(ErrorKind::Serialization)?,
-                ))
-                .await
-                .with_kind(ErrorKind::Network)?;
-        }
-    }
-
-    if !ws_closed {
-        stream
-            .close(Some(CloseFrame {
-                code: CloseCode::Normal,
-                reason: "Log Stream Finished".into(),
-            }))
-            .await
-            .with_kind(ErrorKind::Network)?;
-    }
-
-    Ok(())
-}
-
-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
-#[serde(rename_all = "kebab-case")]
-pub struct LogResponse {
-    entries: Reversible<LogEntry>,
-    start_cursor: Option<String>,
-    end_cursor: Option<String>,
-}
-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
-#[serde(rename_all = "kebab-case")]
-pub struct LogFollowResponse {
-    start_cursor: Option<String>,
-    guid: RequestGuid,
-}
-
-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
-pub struct LogEntry {
-    timestamp: DateTime<Utc>,
-    message: String,
-}
-impl std::fmt::Display for LogEntry {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(
-            f,
-            "{} {}",
-            self.timestamp
-                .to_rfc3339_opts(chrono::SecondsFormat::Millis, true),
-            self.message
-        )
-    }
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct JournalctlEntry {
-    #[serde(rename = "__REALTIME_TIMESTAMP")]
-    pub timestamp: String,
-    #[serde(rename = "MESSAGE")]
-    #[serde(deserialize_with = "deserialize_log_message")]
-    pub message: String,
-    #[serde(rename = "__CURSOR")]
-    pub cursor: String,
-}
-impl JournalctlEntry {
-    fn log_entry(self) -> Result<(String, LogEntry), Error> {
-        Ok((
-            self.cursor,
-            LogEntry {
-                timestamp: DateTime::<Utc>::from(
-                    UNIX_EPOCH + Duration::from_micros(self.timestamp.parse::<u64>()?),
-                ),
-                message: self.message,
-            },
-        ))
-    }
-}
-
-fn deserialize_log_message<'de, D: serde::de::Deserializer<'de>>(
-    deserializer: D,
-) -> std::result::Result<String, D::Error> {
-    struct Visitor;
-    impl<'de> serde::de::Visitor<'de> for Visitor {
-        type Value = String;
-        fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            write!(formatter, "a parsable string")
-        }
-        fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
-        where
-            E: serde::de::Error,
-        {
-            Ok(v.trim().to_owned())
-        }
-        fn visit_unit<E>(self) -> Result<Self::Value, E>
-        where
-            E: serde::de::Error,
-        {
-            Ok(String::new())
-        }
-        fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
-        where
-            A: serde::de::SeqAccess<'de>,
-        {
-            String::from_utf8(
-                std::iter::repeat_with(|| seq.next_element::<u8>().transpose())
-                    .take_while(|a| a.is_some())
-                    .flatten()
-                    .collect::<Result<Vec<u8>, _>>()?,
-            )
-            .map(|s| s.trim().to_owned())
-            .map_err(serde::de::Error::custom)
-        }
-    }
-    deserializer.deserialize_any(Visitor)
-}
-
-/// Defining how we are going to filter on a journalctl cli log.
-/// Kernal: (-k --dmesg                 Show kernel message log from the current boot)
-/// Unit: ( -u --unit=UNIT             Show logs from the specified unit
-///     --user-unit=UNIT        Show logs from the specified user unit))
-/// System: Unit is startd, but we also filter on the comm
-/// Container: Filtering containers, like podman/docker is done by filtering on the CONTAINER_NAME
-#[derive(Debug)]
-pub enum LogSource {
-    Kernel,
-    Unit(&'static str),
-    System,
-    Container(PackageId),
-}
-
-pub const SYSTEM_UNIT: &str = "startd";
-
-#[command(
-    custom_cli(cli_logs(async, context(CliContext))),
-    subcommands(self(logs_nofollow(async)), logs_follow),
-    display(display_none)
-)]
-pub async fn logs(
-    #[arg] id: PackageId,
-    #[arg(short = 'l', long = "limit")] limit: Option<usize>,
-    #[arg(short = 'c', long = "cursor")] cursor: Option<String>,
-    #[arg(short = 'B', long = "before", default)] before: bool,
-    #[arg(short = 'f', long = "follow", default)] follow: bool,
-) -> Result<(PackageId, Option<usize>, Option<String>, bool, bool), Error> {
-    Ok((id, limit, cursor, before, follow))
-}
-pub async fn cli_logs(
-    ctx: CliContext,
-    (id, limit, cursor, before, follow): (PackageId, Option<usize>, Option<String>, bool, bool),
-) -> Result<(), RpcError> {
-    if follow {
-        if cursor.is_some() {
-            return Err(RpcError::from(Error::new(
-                eyre!("The argument '--cursor <cursor>' cannot be used with '--follow'"),
-                crate::ErrorKind::InvalidRequest,
-            )));
-        }
-        if before {
-            return Err(RpcError::from(Error::new(
-                eyre!("The argument '--before' cannot be used with '--follow'"),
-                crate::ErrorKind::InvalidRequest,
-            )));
-        }
-        cli_logs_generic_follow(ctx, "package.logs.follow", Some(id), limit).await
-    } else {
-        cli_logs_generic_nofollow(ctx, "package.logs", Some(id), limit, cursor, before).await
-    }
-}
-pub async fn logs_nofollow(
-    _ctx: (),
-    (id, limit, cursor, before, _): (PackageId, Option<usize>, Option<String>, bool, bool),
-) -> Result<LogResponse, Error> {
-    fetch_logs(LogSource::Container(id), limit, cursor, before).await
-}
-#[command(rpc_only, rename = "follow", display(display_none))]
-pub async fn logs_follow(
-    #[context] ctx: RpcContext,
-    #[parent_data] (id, limit, _, _, _): (PackageId, Option<usize>, Option<String>, bool, bool),
-) -> Result<LogFollowResponse, Error> {
-    follow_logs(ctx, LogSource::Container(id), limit).await
-}
-
-pub async fn cli_logs_generic_nofollow(
-    ctx: CliContext,
-    method: &str,
-    id: Option<PackageId>,
-    limit: Option<usize>,
-    cursor: Option<String>,
-    before: bool,
-) -> Result<(), RpcError> {
-    let res = rpc_toolkit::command_helpers::call_remote(
-        ctx.clone(),
-        method,
-        serde_json::json!({
-            "id": id,
-            "limit": limit,
-            "cursor": cursor,
-            "before": before,
-        }),
-        PhantomData::<LogResponse>,
-    )
-    .await?
-    .result?;
-
-    for entry in res.entries.iter() {
-        println!("{}", entry);
-    }
-
-    Ok(())
-}
-
-pub async fn cli_logs_generic_follow(
-    ctx: CliContext,
-    method: &str,
-    id: Option<PackageId>,
-    limit: Option<usize>,
-) -> Result<(), RpcError> {
-    let res = rpc_toolkit::command_helpers::call_remote(
-        ctx.clone(),
-        method,
-        serde_json::json!({
-            "id": id,
-            "limit": limit,
-        }),
-        PhantomData::<LogFollowResponse>,
-    )
-    .await?
-    .result?;
-
-    let mut base_url = ctx.base_url.clone();
-    let ws_scheme = match base_url.scheme() {
-        "https" => "wss",
-        "http" => "ws",
-        _ => {
-            return Err(Error::new(
-                eyre!("Cannot parse scheme from base URL"),
-                crate::ErrorKind::ParseUrl,
-            )
-            .into())
-        }
-    };
-    base_url
-        .set_scheme(ws_scheme)
-        .map_err(|_| Error::new(eyre!("Cannot set URL scheme"), crate::ErrorKind::ParseUrl))?;
-    let (mut stream, _) =
-                // base_url is "http://127.0.0.1/", with a trailing slash, so we don't put a leading slash in this path:
-                tokio_tungstenite::connect_async(format!("{}ws/rpc/{}", base_url, res.guid)).await?;
-    while let Some(log) = stream.try_next().await? {
-        if let Message::Text(log) = log {
-            println!("{}", serde_json::from_str::<LogEntry>(&log)?);
-        }
-    }
-
-    Ok(())
-}
-
-pub async fn journalctl(
-    id: LogSource,
-    limit: usize,
-    cursor: Option<&str>,
-    before: bool,
-    follow: bool,
-) -> Result<LogStream, Error> {
-    let mut cmd = Command::new("journalctl");
-    cmd.kill_on_drop(true);
-
-    cmd.arg("--output=json");
-    cmd.arg("--output-fields=MESSAGE");
-    cmd.arg(format!("-n{}", limit));
-    match id {
-        LogSource::Kernel => {
-            cmd.arg("-k");
-        }
-        LogSource::Unit(id) => {
-            cmd.arg("-u");
-            cmd.arg(id);
-        }
-        LogSource::System => {
-            cmd.arg("-u");
-            cmd.arg(SYSTEM_UNIT);
-            cmd.arg(format!("_COMM={}", SYSTEM_UNIT));
-        }
-        LogSource::Container(id) => {
-            #[cfg(not(feature = "docker"))]
-            cmd.arg(format!(
-                "SYSLOG_IDENTIFIER={}",
-                DockerProcedure::container_name(&id, None)
-            ));
-            #[cfg(feature = "docker")]
-            cmd.arg(format!(
-                "CONTAINER_NAME={}",
-                DockerProcedure::container_name(&id, None)
-            ));
-        }
-    };
-
-    let cursor_formatted = format!("--after-cursor={}", cursor.unwrap_or(""));
-    if cursor.is_some() {
-        cmd.arg(&cursor_formatted);
-        if before {
-            cmd.arg("--reverse");
-        }
-    }
-    if follow {
-        cmd.arg("--follow");
-    }
-
-    let mut child = cmd.stdout(Stdio::piped()).spawn()?;
-    let out = BufReader::new(
-        child
-            .stdout
-            .take()
-            .ok_or_else(|| Error::new(eyre!("No stdout available"), crate::ErrorKind::Journald))?,
-    );
-
-    let journalctl_entries = LinesStream::new(out.lines());
-
-    let deserialized_entries = journalctl_entries
-        .map_err(|e| Error::new(e, crate::ErrorKind::Journald))
-        .and_then(|s| {
-            futures::future::ready(
-                serde_json::from_str::<JournalctlEntry>(&s)
-                    .with_kind(crate::ErrorKind::Deserialization),
-            )
-        });
-
-    Ok(LogStream {
-        _child: child,
-        entries: deserialized_entries.boxed(),
-    })
-}
-
-#[instrument(skip_all)]
-pub async fn fetch_logs(
-    id: LogSource,
-    limit: Option<usize>,
-    cursor: Option<String>,
-    before: bool,
-) -> Result<LogResponse, Error> {
-    let limit = limit.unwrap_or(50);
-    let mut stream = journalctl(id, limit, cursor.as_deref(), before, false).await?;
-
-    let mut entries = Vec::with_capacity(limit);
-    let mut start_cursor = None;
-
-    if let Some(first) = tokio::time::timeout(Duration::from_secs(1), stream.try_next())
-        .await
-        .ok()
-        .transpose()?
-        .flatten()
-    {
-        let (cursor, entry) = first.log_entry()?;
-        start_cursor = Some(cursor);
-        entries.push(entry);
-    }
-
-    let (mut end_cursor, entries) = stream
-        .try_fold(
-            (start_cursor.clone(), entries),
-            |(_, mut acc), entry| async move {
-                let (cursor, entry) = entry.log_entry()?;
-                acc.push(entry);
-                Ok((Some(cursor), acc))
-            },
-        )
-        .await?;
-    let mut entries = Reversible::new(entries);
-    // reverse again so output is always in increasing chronological order
-    if cursor.is_some() && before {
-        entries.reverse();
-        std::mem::swap(&mut start_cursor, &mut end_cursor);
-    }
-    Ok(LogResponse {
-        entries,
-        start_cursor,
-        end_cursor,
-    })
-}
-
-#[instrument(skip_all)]
-pub async fn follow_logs(
-    ctx: RpcContext,
-    id: LogSource,
-    limit: Option<usize>,
-) -> Result<LogFollowResponse, Error> {
-    let limit = limit.unwrap_or(50);
-    let mut stream = journalctl(id, limit, None, false, true).await?;
-
-    let mut start_cursor = None;
-    let mut first_entry = None;
-
-    if let Some(first) = tokio::time::timeout(Duration::from_secs(1), stream.try_next())
-        .await
-        .ok()
-        .transpose()?
-        .flatten()
-    {
-        let (cursor, entry) = first.log_entry()?;
-        start_cursor = Some(cursor);
-        first_entry = Some(entry);
-    }
-
-    let guid = RequestGuid::new();
-    ctx.add_continuation(
-        guid.clone(),
-        RpcContinuation::ws(
-            Box::new(move |ws_fut| ws_handler(first_entry, stream, ws_fut).boxed()),
-            Duration::from_secs(30),
-        ),
-    )
-    .await;
-    Ok(LogFollowResponse { start_cursor, guid })
-}
-
-// #[tokio::test]
-// pub async fn test_logs() {
-//     let response = fetch_logs(
-//         // change `tor.service` to an actual journald unit on your machine
-//         // LogSource::Service("tor.service"),
-//         // first run `docker run --name=hello-world.embassy --log-driver=journald hello-world`
-//         LogSource::Container("hello-world".parse().unwrap()),
-//         // Some(5),
-//         None,
-//         None,
-//         // Some("s=1b8c418e28534400856c27b211dd94fd;i=5a7;b=97571c13a1284f87bc0639b5cff5acbe;m=740e916;t=5ca073eea3445;x=f45bc233ca328348".to_owned()),
-//         false,
-//         true,
-//     )
-//     .await
-//     .unwrap();
-//     let serialized = serde_json::to_string_pretty(&response).unwrap();
-//     println!("{}", serialized);
-// }
-
-// #[tokio::test]
-// pub async fn test_logs() {
-//     let mut cmd = Command::new("journalctl");
-//     cmd.kill_on_drop(true);
-
-//     cmd.arg("-f");
-//     cmd.arg("CONTAINER_NAME=hello-world.embassy");
-
-//     let mut child = cmd.stdout(Stdio::piped()).spawn().unwrap();
-//     let out = BufReader::new(
-//         child
-//             .stdout
-//             .take()
-//             .ok_or_else(|| Error::new(eyre!("No stdout available"), crate::ErrorKind::Journald))
-//             .unwrap(),
-//     );
-
-//     let mut journalctl_entries = LinesStream::new(out.lines());
-
-//     while let Some(line) = journalctl_entries.try_next().await.unwrap() {
-//         dbg!(line);
-//     }
-// }