rename frontend to web and update contributing guide (#2509)

* rename frontend to web and update contributing guide

* rename this time

* fix build

* restructure rust code

* update documentation

* update descriptions

* Update CONTRIBUTING.md

Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>

core/startos/.sqlx/query-1ce5254f27de971fd87f5ab66d300f2b22433c86617a0dbf796bf2170186dd2e.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Bytea"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "1ce5254f27de971fd87f5ab66d300f2b22433c86617a0dbf796bf2170186dd2e"
+}

core/startos/.sqlx/query-21471490cdc3adb206274cc68e1ea745ffa5da4479478c1fd2158a45324b1930.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM ssh_keys WHERE fingerprint = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "21471490cdc3adb206274cc68e1ea745ffa5da4479478c1fd2158a45324b1930"
+}

core/startos/.sqlx/query-28ea34bbde836e0618c5fc9bb7c36e463c20c841a7d6a0eb15be0f24f4a928ec.json

@@ -0,0 +1,40 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT hostname, path, username, password FROM cifs_shares WHERE id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "hostname",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "username",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "password",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "28ea34bbde836e0618c5fc9bb7c36e463c20c841a7d6a0eb15be0f24f4a928ec"
+}

core/startos/.sqlx/query-4099028a5c0de578255bf54a67cef6cb0f1e9a4e158260700f1639dd4b438997.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM ssh_keys WHERE fingerprint = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "fingerprint",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "openssh_pubkey",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "created_at",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "4099028a5c0de578255bf54a67cef6cb0f1e9a4e158260700f1639dd4b438997"
+}

core/startos/.sqlx/query-4691e3a2ce80b59009ac17124f54f925f61dc5ea371903e62cdffa5d7b67ca96.json

@@ -0,0 +1,50 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM session WHERE logged_out IS NULL OR logged_out > CURRENT_TIMESTAMP",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "logged_in",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 2,
+        "name": "logged_out",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 3,
+        "name": "last_active",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 4,
+        "name": "user_agent",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 5,
+        "name": "metadata",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      false,
+      true,
+      false
+    ]
+  },
+  "hash": "4691e3a2ce80b59009ac17124f54f925f61dc5ea371903e62cdffa5d7b67ca96"
+}

core/startos/.sqlx/query-4bcfbefb1eb3181343871a1cd7fc3afb81c2be5c681cfa8b4be0ce70610e9c3a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE session SET logged_out = CURRENT_TIMESTAMP WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "4bcfbefb1eb3181343871a1cd7fc3afb81c2be5c681cfa8b4be0ce70610e9c3a"
+}

core/startos/.sqlx/query-629be61c3c341c131ddbbff0293a83dbc6afd07cae69d246987f62cf0cc35c2a.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT password FROM account",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "password",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "629be61c3c341c131ddbbff0293a83dbc6afd07cae69d246987f62cf0cc35c2a"
+}

core/startos/.sqlx/query-687688055e63d27123cdc89a5bbbd8361776290a9411d527eaf1fdb40bef399d.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT key FROM tor WHERE package = $1 AND interface = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "key",
+        "type_info": "Bytea"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "687688055e63d27123cdc89a5bbbd8361776290a9411d527eaf1fdb40bef399d"
+}

core/startos/.sqlx/query-6d35ccf780fb2bb62586dd1d3df9c1550a41ee580dad3f49d35cb843ebef10ca.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE session SET last_active = CURRENT_TIMESTAMP WHERE id = $1 AND logged_out IS NULL OR logged_out > CURRENT_TIMESTAMP",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "6d35ccf780fb2bb62586dd1d3df9c1550a41ee580dad3f49d35cb843ebef10ca"
+}

core/startos/.sqlx/query-770c1017734720453dc87b58c385b987c5af5807151ff71a59000014586752e0.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO UPDATE SET package = EXCLUDED.package RETURNING key",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "key",
+        "type_info": "Bytea"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Bytea"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "770c1017734720453dc87b58c385b987c5af5807151ff71a59000014586752e0"
+}

core/startos/.sqlx/query-7b64f032d507e8ffe37c41f4c7ad514a66c421a11ab04c26d89a7aa8f6b67210.json

@@ -0,0 +1,65 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, package_id, created_at, code, level, title, message, data FROM notifications WHERE id < $1 ORDER BY id DESC LIMIT $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "package_id",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "created_at",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 3,
+        "name": "code",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "level",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 5,
+        "name": "title",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 6,
+        "name": "message",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 7,
+        "name": "data",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "7b64f032d507e8ffe37c41f4c7ad514a66c421a11ab04c26d89a7aa8f6b67210"
+}

core/startos/.sqlx/query-7c7a3549c997eb75bf964ea65fbb98a73045adf618696cd838d79203ef5383fb.json

@@ -0,0 +1,19 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            INSERT INTO account (\n                id,\n                server_id,\n                hostname,\n                password,\n                network_key,\n                root_ca_key_pem,\n                root_ca_cert_pem\n            ) VALUES (\n                0, $1, $2, $3, $4, $5, $6\n            ) ON CONFLICT (id) DO UPDATE SET\n                server_id = EXCLUDED.server_id,\n                hostname = EXCLUDED.hostname,\n                password = EXCLUDED.password,\n                network_key = EXCLUDED.network_key,\n                root_ca_key_pem = EXCLUDED.root_ca_key_pem,\n                root_ca_cert_pem = EXCLUDED.root_ca_cert_pem\n            ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text",
+        "Bytea",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "7c7a3549c997eb75bf964ea65fbb98a73045adf618696cd838d79203ef5383fb"
+}

core/startos/.sqlx/query-7e0649d839927e57fa03ee51a2c9f96a8bdb0fc97ee8a3c6df1069e1e2b98576.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM tor WHERE package = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "7e0649d839927e57fa03ee51a2c9f96a8bdb0fc97ee8a3c6df1069e1e2b98576"
+}

core/startos/.sqlx/query-8951b9126fbf60dbb5997241e11e3526b70bccf3e407327917294a993bc17ed5.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Bytea"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8951b9126fbf60dbb5997241e11e3526b70bccf3e407327917294a993bc17ed5"
+}

core/startos/.sqlx/query-94d471bb374b4965c6cbedf8c17bbf6bea226d38efaf6559923c79a36d5ca08c.json

@@ -0,0 +1,64 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, package_id, created_at, code, level, title, message, data FROM notifications ORDER BY id DESC LIMIT $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "package_id",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "created_at",
+        "type_info": "Timestamp"
+      },
+      {
+        "ordinal": 3,
+        "name": "code",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 4,
+        "name": "level",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 5,
+        "name": "title",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 6,
+        "name": "message",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 7,
+        "name": "data",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "94d471bb374b4965c6cbedf8c17bbf6bea226d38efaf6559923c79a36d5ca08c"
+}

core/startos/.sqlx/query-95c4ab4c645f3302568c6ff13d85ab58252362694cf0f56999bf60194d20583a.json

@@ -0,0 +1,44 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id, hostname, path, username, password FROM cifs_shares",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "hostname",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "path",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "username",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "password",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true
+    ]
+  },
+  "hash": "95c4ab4c645f3302568c6ff13d85ab58252362694cf0f56999bf60194d20583a"
+}

core/startos/.sqlx/query-a60d6e66719325b08dc4ecfacaf337527233c84eee758ac9be967906e5841d27.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM cifs_shares WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "a60d6e66719325b08dc4ecfacaf337527233c84eee758ac9be967906e5841d27"
+}

core/startos/.sqlx/query-a6b0c8909a3a5d6d9156aebfb359424e6b5a1d1402e028219e21726f1ebd282e.json

@@ -0,0 +1,32 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT fingerprint, openssh_pubkey, created_at FROM ssh_keys",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "fingerprint",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "openssh_pubkey",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "created_at",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "a6b0c8909a3a5d6d9156aebfb359424e6b5a1d1402e028219e21726f1ebd282e"
+}

core/startos/.sqlx/query-b1147beaaabbed89f2ab8c1e13ec4393a9a8fde2833cf096af766a979d94dee6.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE cifs_shares SET hostname = $1, path = $2, username = $3, password = $4 WHERE id = $5",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text",
+        "Text",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b1147beaaabbed89f2ab8c1e13ec4393a9a8fde2833cf096af766a979d94dee6"
+}

core/startos/.sqlx/query-b203820ee1c553a4b246eac74b79bd10d5717b2a0ddecf22330b7d531aac7c5d.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM network_keys WHERE package = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b203820ee1c553a4b246eac74b79bd10d5717b2a0ddecf22330b7d531aac7c5d"
+}

core/startos/.sqlx/query-d5117054072476377f3c4f040ea429d4c9b2cf534e76f35c80a2bf60e8599cca.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT openssh_pubkey FROM ssh_keys",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "openssh_pubkey",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "d5117054072476377f3c4f040ea429d4c9b2cf534e76f35c80a2bf60e8599cca"
+}

core/startos/.sqlx/query-da71f94b29798d1738d2b10b9a721ea72db8cfb362e7181c8226d9297507c62b.json

@@ -0,0 +1,19 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO notifications (package_id, code, level, title, message, data) VALUES ($1, $2, $3, $4, $5, $6)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int4",
+        "Text",
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "da71f94b29798d1738d2b10b9a721ea72db8cfb362e7181c8226d9297507c62b"
+}

core/startos/.sqlx/query-e185203cf84e43b801dfb23b4159e34aeaef1154dcd3d6811ab504915497ccf7.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM notifications WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e185203cf84e43b801dfb23b4159e34aeaef1154dcd3d6811ab504915497ccf7"
+}

core/startos/.sqlx/query-e545696735f202f9d13cf22a561f3ff3f9aed7f90027a9ba97634bcb47d772f0.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT tor_key FROM account WHERE id = 0",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "tor_key",
+        "type_info": "Bytea"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "e545696735f202f9d13cf22a561f3ff3f9aed7f90027a9ba97634bcb47d772f0"
+}

core/startos/.sqlx/query-e5843c5b0e7819b29aa1abf2266799bd4f82e761837b526a0972c3d4439a264d.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO session (id, user_agent, metadata) VALUES ($1, $2, $3)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e5843c5b0e7819b29aa1abf2266799bd4f82e761837b526a0972c3d4439a264d"
+}

core/startos/.sqlx/query-e95322a8e2ae3b93f1e974b24c0b81803f1e9ec9e8ebbf15cafddfc1c5a028ed.json

@@ -0,0 +1,40 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                SELECT\n                    network_keys.package,\n                    network_keys.interface,\n                    network_keys.key,\n                    tor.key AS \"tor_key?\"\n                FROM\n                    network_keys\n                LEFT JOIN\n                    tor\n                ON\n                    network_keys.package = tor.package\n                AND\n                    network_keys.interface = tor.interface\n                WHERE\n                    network_keys.package = $1\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "package",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "interface",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "key",
+        "type_info": "Bytea"
+      },
+      {
+        "ordinal": 3,
+        "name": "tor_key?",
+        "type_info": "Bytea"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "e95322a8e2ae3b93f1e974b24c0b81803f1e9ec9e8ebbf15cafddfc1c5a028ed"
+}

core/startos/.sqlx/query-eb750adaa305bdbf3c5b70aaf59139c7b7569602adb58f2d6b3a94da4f167b0a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM notifications WHERE id < $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "eb750adaa305bdbf3c5b70aaf59139c7b7569602adb58f2d6b3a94da4f167b0a"
+}

core/startos/.sqlx/query-ecc765d8205c0876956f95f76944ac6a5f34dd820c4073b7728c7067aab9fded.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO cifs_shares (hostname, path, username, password) VALUES ($1, $2, $3, $4) RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "ecc765d8205c0876956f95f76944ac6a5f34dd820c4073b7728c7067aab9fded"
+}

core/startos/.sqlx/query-f6d1c5ef0f9d9577bea8382318967b9deb46da75788c7fe6082b43821c22d556.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO ssh_keys (fingerprint, openssh_pubkey, created_at) VALUES ($1, $2, $3)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f6d1c5ef0f9d9577bea8382318967b9deb46da75788c7fe6082b43821c22d556"
+}

core/startos/.sqlx/query-f7d2dae84613bcef330f7403352cc96547f3f6dbec11bf2eadfaf53ad8ab51b5.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT network_key FROM account WHERE id = 0",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "network_key",
+        "type_info": "Bytea"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "f7d2dae84613bcef330f7403352cc96547f3f6dbec11bf2eadfaf53ad8ab51b5"
+}

core/startos/.sqlx/query-fe6e4f09f3028e5b6b6259e86cbad285680ce157aae9d7837ac020c8b2945e7f.json

@@ -0,0 +1,62 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * FROM account WHERE id = 0",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 1,
+        "name": "password",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "tor_key",
+        "type_info": "Bytea"
+      },
+      {
+        "ordinal": 3,
+        "name": "server_id",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "hostname",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 5,
+        "name": "network_key",
+        "type_info": "Bytea"
+      },
+      {
+        "ordinal": 6,
+        "name": "root_ca_key_pem",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 7,
+        "name": "root_ca_cert_pem",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      true,
+      true,
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "fe6e4f09f3028e5b6b6259e86cbad285680ce157aae9d7837ac020c8b2945e7f"
+}

core/startos/Cargo.toml

@@ -0,0 +1,176 @@
+[package]
+authors = ["Aiden McClelland <me@drbonez.dev>"]
+description = "The core of StartOS"
+documentation = "https://docs.rs/start-os"
+edition = "2021"
+keywords = [
+  "self-hosted",
+  "raspberry-pi",
+  "privacy",
+  "bitcoin",
+  "full-node",
+  "lightning",
+]
+name = "start-os"
+readme = "README.md"
+repository = "https://github.com/Start9Labs/start-os"
+version = "0.3.5"
+license = "MIT"
+
+[lib]
+name = "startos"
+path = "src/lib.rs"
+
+[[bin]]
+name = "startbox"
+path = "src/main.rs"
+
+[features]
+avahi = ["avahi-sys"]
+avahi-alias = ["avahi"]
+cli = []
+daemon = []
+default = ["cli", "sdk", "daemon", "js-engine"]
+dev = []
+docker = []
+sdk = []
+unstable = ["console-subscriber", "tokio/tracing"]
+
+[dependencies]
+aes = { version = "0.7.5", features = ["ctr"] }
+async-compression = { version = "0.4.4", features = [
+  "gzip",
+  "brotli",
+  "tokio",
+] }
+async-stream = "0.3.5"
+async-trait = "0.1.74"
+avahi-sys = { git = "https://github.com/Start9Labs/avahi-sys", version = "0.10.0", branch = "feature/dynamic-linking", features = [
+  "dynamic",
+], optional = true }
+base32 = "0.4.0"
+base64 = "0.21.4"
+base64ct = "1.6.0"
+basic-cookies = "0.1.4"
+bytes = "1"
+chrono = { version = "0.4.31", features = ["serde"] }
+clap = "3.2.25"
+color-eyre = "0.6.2"
+console = "0.15.7"
+console-subscriber = { version = "0.2", optional = true }
+cookie = "0.18.0"
+cookie_store = "0.20.0"
+current_platform = "0.2.0"
+digest = "0.10.7"
+divrem = "1.0.0"
+ed25519 = { version = "2.2.3", features = ["pkcs8", "pem", "alloc"] }
+ed25519-dalek = { version = "2.0.0", features = [
+  "serde",
+  "zeroize",
+  "rand_core",
+  "digest",
+] }
+ed25519-dalek-v1 = { package = "ed25519-dalek", version = "1" }
+container-init = { path = "../container-init" }
+emver = { version = "0.1.7", git = "https://github.com/Start9Labs/emver-rs.git", features = [
+  "serde",
+] }
+fd-lock-rs = "0.1.4"
+futures = "0.3.28"
+gpt = "3.1.0"
+helpers = { path = "../helpers" }
+hex = "0.4.3"
+hmac = "0.12.1"
+http = "0.2.9"
+hyper = { version = "0.14.27", features = ["full"] }
+hyper-ws-listener = "0.3.0"
+imbl = "2.0.2"
+imbl-value = { git = "https://github.com/Start9Labs/imbl-value.git" }
+include_dir = "0.7.3"
+indexmap = { version = "2.0.2", features = ["serde"] }
+indicatif = { version = "0.17.7", features = ["tokio"] }
+ipnet = { version = "2.8.0", features = ["serde"] }
+iprange = { version = "0.6.7", features = ["serde"] }
+isocountry = "0.3.2"
+itertools = "0.11.0"
+jaq-core = "0.10.1"
+jaq-std = "0.10.0"
+josekit = "0.8.4"
+js-engine = { path = '../js-engine', optional = true }
+jsonpath_lib = { git = "https://github.com/Start9Labs/jsonpath.git" }
+lazy_static = "1.4.0"
+libc = "0.2.149"
+log = "0.4.20"
+mbrman = "0.5.2"
+models = { version = "*", path = "../models" }
+new_mime_guess = "4"
+nix = { version = "0.27.1", features = ["user", "process", "signal", "fs"] }
+nom = "7.1.3"
+num = "0.4.1"
+num_enum = "0.7.0"
+openssh-keys = "0.6.2"
+openssl = { version = "0.10.57", features = ["vendored"] }
+p256 = { version = "0.13.2", features = ["pem"] }
+patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
+  "trace",
+] }
+pbkdf2 = "0.12.2"
+pin-project = "1.1.3"
+pkcs8 = { version = "0.10.2", features = ["std"] }
+prettytable-rs = "0.10.0"
+proptest = "1.3.1"
+proptest-derive = "0.4.0"
+rand = { version = "0.8.5", features = ["std"] }
+regex = "1.10.2"
+reqwest = { version = "0.11.22", features = ["stream", "json", "socks"] }
+reqwest_cookie_store = "0.6.0"
+rpassword = "7.2.0"
+rpc-toolkit = "0.2.2"
+rust-argon2 = "2.0.0"
+scopeguard = "1.1" # because avahi-sys fucks your shit up
+serde = { version = "1.0", features = ["derive", "rc"] }
+serde_cbor = { package = "ciborium", version = "0.2.1" }
+serde_json = "1.0"
+serde_toml = { package = "toml", version = "0.8.2" }
+serde_with = { version = "3.4.0", features = ["macros", "json"] }
+serde_yaml = "0.9.25"
+sha2 = "0.10.2"
+simple-logging = "2.0.2"
+sqlx = { version = "0.7.2", features = [
+  "chrono",
+  "runtime-tokio-rustls",
+  "postgres",
+] }
+sscanf = "0.4.1"
+ssh-key = { version = "0.6.2", features = ["ed25519"] }
+stderrlog = "0.5.4"
+tar = "0.4.40"
+thiserror = "1.0.49"
+tokio = { version = "1", features = ["full"] }
+tokio-rustls = "0.24.1"
+tokio-socks = "0.5.1"
+tokio-stream = { version = "0.1.14", features = ["io-util", "sync", "net"] }
+tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
+tokio-tungstenite = { version = "0.20.1", features = ["native-tls"] }
+tokio-util = { version = "0.7.9", features = ["io"] }
+torut = "0.2.1"
+tracing = "0.1.39"
+tracing-error = "0.2.0"
+tracing-futures = "0.2.5"
+tracing-journald = "0.3.0"
+tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
+trust-dns-server = "0.23.1"
+typed-builder = "0.17.0"
+url = { version = "2.4.1", features = ["serde"] }
+urlencoding = "2.1.3"
+uuid = { version = "1.4.1", features = ["v4"] }
+zeroize = "1.6.0"
+
+[profile.test]
+opt-level = 3
+
+[profile.dev.package.backtrace]
+opt-level = 3
+
+[profile.dev.package.sqlx-macros]
+opt-level = 3

core/startos/deny.toml

@@ -0,0 +1,22 @@
+[licenses]
+unlicensed = "warn"
+allow-osi-fsf-free = "neither"
+copyleft = "deny"
+confidence-threshold = 0.93
+allow = [
+    "Apache-2.0",
+    "Apache-2.0 WITH LLVM-exception",
+    "MIT",
+    "ISC",
+    "MPL-2.0",
+    "CC0-1.0",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "LGPL-3.0",
+    "OpenSSL",
+]
+
+clarify = [
+    { name = "webpki", expression = "ISC", license-files = [ { path = "LICENSE", hash = 0x001c7e6c } ] },
+    { name = "ring", expression = "OpenSSL", license-files = [ { path = "LICENSE", hash = 0xbd0eed23 } ] },
+]

core/startos/migrations/20210629193146_Init.sql

@@ -0,0 +1,60 @@
+-- Add migration script here
+CREATE TABLE IF NOT EXISTS tor (
+    package TEXT NOT NULL,
+    interface TEXT NOT NULL,
+    key BYTEA NOT NULL CHECK (length(key) = 64),
+    PRIMARY KEY (package, interface)
+);
+
+CREATE TABLE IF NOT EXISTS session (
+    id TEXT NOT NULL PRIMARY KEY,
+    logged_in TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    logged_out TIMESTAMP,
+    last_active TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    user_agent TEXT,
+    metadata TEXT NOT NULL DEFAULT 'null'
+);
+
+CREATE TABLE IF NOT EXISTS account (
+    id SERIAL PRIMARY KEY CHECK (id = 0),
+    password TEXT NOT NULL,
+    tor_key BYTEA NOT NULL CHECK (length(tor_key) = 64)
+);
+
+CREATE TABLE IF NOT EXISTS ssh_keys (
+    fingerprint TEXT NOT NULL,
+    openssh_pubkey TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    PRIMARY KEY (fingerprint)
+);
+
+CREATE TABLE IF NOT EXISTS certificates (
+    id SERIAL PRIMARY KEY,
+    -- Root = 0, Int = 1, Other = 2..
+    priv_key_pem TEXT NOT NULL,
+    certificate_pem TEXT NOT NULL,
+    lookup_string TEXT UNIQUE,
+    created_at TEXT,
+    updated_at TEXT
+);
+
+ALTER SEQUENCE certificates_id_seq START 2 RESTART 2;
+
+CREATE TABLE IF NOT EXISTS notifications (
+    id SERIAL PRIMARY KEY,
+    package_id TEXT,
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    code INTEGER NOT NULL,
+    level TEXT NOT NULL,
+    title TEXT NOT NULL,
+    message TEXT NOT NULL,
+    data TEXT
+);
+
+CREATE TABLE IF NOT EXISTS cifs_shares (
+    id SERIAL PRIMARY KEY,
+    hostname TEXT NOT NULL,
+    path TEXT NOT NULL,
+    username TEXT NOT NULL,
+    password TEXT
+);

core/startos/migrations/20230118185232_NetworkKeys.sql

@@ -0,0 +1,62 @@
+-- Add migration script here
+CREATE EXTENSION pgcrypto;
+
+ALTER TABLE
+    account
+ADD
+    COLUMN server_id TEXT,
+ADD
+    COLUMN hostname TEXT,
+ADD
+    COLUMN network_key BYTEA CHECK (length(network_key) = 32),
+ADD
+    COLUMN root_ca_key_pem TEXT,
+ADD
+    COLUMN root_ca_cert_pem TEXT;
+
+UPDATE
+    account
+SET
+    network_key = gen_random_bytes(32),
+    root_ca_key_pem = (
+        SELECT
+            priv_key_pem
+        FROM
+            certificates
+        WHERE
+            id = 0
+    ),
+    root_ca_cert_pem = (
+        SELECT
+            certificate_pem
+        FROM
+            certificates
+        WHERE
+            id = 0
+    )
+WHERE
+    id = 0;
+
+ALTER TABLE
+    account
+ALTER COLUMN
+    tor_key DROP NOT NULL,
+ALTER COLUMN
+    network_key
+SET
+    NOT NULL,
+ALTER COLUMN
+    root_ca_key_pem
+SET
+    NOT NULL,
+ALTER COLUMN
+    root_ca_cert_pem
+SET
+    NOT NULL;
+
+CREATE TABLE IF NOT EXISTS network_keys (
+    package TEXT NOT NULL,
+    interface TEXT NOT NULL,
+    key BYTEA NOT NULL CHECK (length(key) = 32),
+    PRIMARY KEY (package, interface)
+);

core/startos/src/account.rs

@@ -0,0 +1,132 @@
+use std::time::SystemTime;
+
+use ed25519_dalek::SecretKey;
+use openssl::pkey::{PKey, Private};
+use openssl::x509::X509;
+use sqlx::PgExecutor;
+
+use crate::hostname::{generate_hostname, generate_id, Hostname};
+use crate::net::keys::Key;
+use crate::net::ssl::{generate_key, make_root_cert};
+use crate::prelude::*;
+use crate::util::crypto::ed25519_expand_key;
+
+fn hash_password(password: &str) -> Result<String, Error> {
+    argon2::hash_encoded(
+        password.as_bytes(),
+        &rand::random::<[u8; 16]>()[..],
+        &argon2::Config::rfc9106_low_mem(),
+    )
+    .with_kind(crate::ErrorKind::PasswordHashGeneration)
+}
+
+#[derive(Debug, Clone)]
+pub struct AccountInfo {
+    pub server_id: String,
+    pub hostname: Hostname,
+    pub password: String,
+    pub key: Key,
+    pub root_ca_key: PKey<Private>,
+    pub root_ca_cert: X509,
+}
+impl AccountInfo {
+    pub fn new(password: &str, start_time: SystemTime) -> Result<Self, Error> {
+        let server_id = generate_id();
+        let hostname = generate_hostname();
+        let root_ca_key = generate_key()?;
+        let root_ca_cert = make_root_cert(&root_ca_key, &hostname, start_time)?;
+        Ok(Self {
+            server_id,
+            hostname,
+            password: hash_password(password)?,
+            key: Key::new(None),
+            root_ca_key,
+            root_ca_cert,
+        })
+    }
+
+    pub async fn load(secrets: impl PgExecutor<'_>) -> Result<Self, Error> {
+        let r = sqlx::query!("SELECT * FROM account WHERE id = 0")
+            .fetch_one(secrets)
+            .await?;
+
+        let server_id = r.server_id.unwrap_or_else(generate_id);
+        let hostname = r.hostname.map(Hostname).unwrap_or_else(generate_hostname);
+        let password = r.password;
+        let network_key = SecretKey::try_from(r.network_key).map_err(|e| {
+            Error::new(
+                eyre!("expected vec of len 32, got len {}", e.len()),
+                ErrorKind::ParseDbField,
+            )
+        })?;
+        let tor_key = if let Some(k) = &r.tor_key {
+            <[u8; 64]>::try_from(&k[..]).map_err(|_| {
+                Error::new(
+                    eyre!("expected vec of len 64, got len {}", k.len()),
+                    ErrorKind::ParseDbField,
+                )
+            })?
+        } else {
+            ed25519_expand_key(&network_key)
+        };
+        let key = Key::from_pair(None, network_key, tor_key);
+        let root_ca_key = PKey::private_key_from_pem(r.root_ca_key_pem.as_bytes())?;
+        let root_ca_cert = X509::from_pem(r.root_ca_cert_pem.as_bytes())?;
+
+        Ok(Self {
+            server_id,
+            hostname,
+            password,
+            key,
+            root_ca_key,
+            root_ca_cert,
+        })
+    }
+
+    pub async fn save(&self, secrets: impl PgExecutor<'_>) -> Result<(), Error> {
+        let server_id = self.server_id.as_str();
+        let hostname = self.hostname.0.as_str();
+        let password = self.password.as_str();
+        let network_key = self.key.as_bytes();
+        let network_key = network_key.as_slice();
+        let root_ca_key = String::from_utf8(self.root_ca_key.private_key_to_pem_pkcs8()?)?;
+        let root_ca_cert = String::from_utf8(self.root_ca_cert.to_pem()?)?;
+
+        sqlx::query!(
+            r#"
+            INSERT INTO account (
+                id,
+                server_id,
+                hostname,
+                password,
+                network_key,
+                root_ca_key_pem,
+                root_ca_cert_pem
+            ) VALUES (
+                0, $1, $2, $3, $4, $5, $6
+            ) ON CONFLICT (id) DO UPDATE SET
+                server_id = EXCLUDED.server_id,
+                hostname = EXCLUDED.hostname,
+                password = EXCLUDED.password,
+                network_key = EXCLUDED.network_key,
+                root_ca_key_pem = EXCLUDED.root_ca_key_pem,
+                root_ca_cert_pem = EXCLUDED.root_ca_cert_pem
+            "#,
+            server_id,
+            hostname,
+            password,
+            network_key,
+            root_ca_key,
+            root_ca_cert,
+        )
+        .execute(secrets)
+        .await?;
+
+        Ok(())
+    }
+
+    pub fn set_password(&mut self, password: &str) -> Result<(), Error> {
+        self.password = hash_password(password)?;
+        Ok(())
+    }
+}

core/startos/src/action.rs

@@ -0,0 +1,163 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use indexmap::IndexSet;
+pub use models::ActionId;
+use models::ImageId;
+use rpc_toolkit::command;
+use serde::{Deserialize, Serialize};
+use tracing::instrument;
+
+use crate::config::{Config, ConfigSpec};
+use crate::context::RpcContext;
+use crate::prelude::*;
+use crate::procedure::docker::DockerContainers;
+use crate::procedure::{PackageProcedure, ProcedureName};
+use crate::s9pk::manifest::PackageId;
+use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
+use crate::util::Version;
+use crate::volume::Volumes;
+use crate::{Error, ResultExt};
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct Actions(pub BTreeMap<ActionId, Action>);
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "version")]
+pub enum ActionResult {
+    #[serde(rename = "0")]
+    V0(ActionResultV0),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ActionResultV0 {
+    pub message: String,
+    pub value: Option<String>,
+    pub copyable: bool,
+    pub qr: bool,
+}
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum DockerStatus {
+    Running,
+    Stopped,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Action {
+    pub name: String,
+    pub description: String,
+    #[serde(default)]
+    pub warning: Option<String>,
+    pub implementation: PackageProcedure,
+    pub allowed_statuses: IndexSet<DockerStatus>,
+    #[serde(default)]
+    pub input_spec: ConfigSpec,
+}
+impl Action {
+    #[instrument(skip_all)]
+    pub fn validate(
+        &self,
+        _container: &Option<DockerContainers>,
+        eos_version: &Version,
+        volumes: &Volumes,
+        image_ids: &BTreeSet<ImageId>,
+    ) -> Result<(), Error> {
+        self.implementation
+            .validate(eos_version, volumes, image_ids, true)
+            .with_ctx(|_| {
+                (
+                    crate::ErrorKind::ValidateS9pk,
+                    format!("Action {}", self.name),
+                )
+            })
+    }
+
+    #[instrument(skip_all)]
+    pub async fn execute(
+        &self,
+        ctx: &RpcContext,
+        pkg_id: &PackageId,
+        pkg_version: &Version,
+        action_id: &ActionId,
+        volumes: &Volumes,
+        input: Option<Config>,
+    ) -> Result<ActionResult, Error> {
+        if let Some(ref input) = input {
+            self.input_spec
+                .matches(&input)
+                .with_kind(crate::ErrorKind::ConfigSpecViolation)?;
+        }
+        self.implementation
+            .execute(
+                ctx,
+                pkg_id,
+                pkg_version,
+                ProcedureName::Action(action_id.clone()),
+                volumes,
+                input,
+                None,
+            )
+            .await?
+            .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::Action))
+    }
+}
+
+fn display_action_result(action_result: ActionResult, matches: &ArgMatches) {
+    if matches.is_present("format") {
+        return display_serializable(action_result, matches);
+    }
+    match action_result {
+        ActionResult::V0(ar) => {
+            println!(
+                "{}: {}",
+                ar.message,
+                serde_json::to_string(&ar.value).unwrap()
+            );
+        }
+    }
+}
+
+#[command(about = "Executes an action", display(display_action_result))]
+#[instrument(skip_all)]
+pub async fn action(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "id")] pkg_id: PackageId,
+    #[arg(rename = "action-id")] action_id: ActionId,
+    #[arg(stdin, parse(parse_stdin_deserializable))] input: Option<Config>,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+) -> Result<ActionResult, Error> {
+    let manifest = ctx
+        .db
+        .peek()
+        .await
+        .as_package_data()
+        .as_idx(&pkg_id)
+        .or_not_found(&pkg_id)?
+        .as_installed()
+        .or_not_found(&pkg_id)?
+        .as_manifest()
+        .de()?;
+
+    if let Some(action) = manifest.actions.0.get(&action_id) {
+        action
+            .execute(
+                &ctx,
+                &manifest.id,
+                &manifest.version,
+                &action_id,
+                &manifest.volumes,
+                input,
+            )
+            .await
+    } else {
+        Err(Error::new(
+            eyre!("Action not found in manifest"),
+            crate::ErrorKind::NotFound,
+        ))
+    }
+}

core/startos/src/auth.rs

@@ -0,0 +1,391 @@
+use std::collections::BTreeMap;
+use std::marker::PhantomData;
+
+use chrono::{DateTime, Utc};
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use josekit::jwk::Jwk;
+use rpc_toolkit::command;
+use rpc_toolkit::command_helpers::prelude::{RequestParts, ResponseParts};
+use rpc_toolkit::yajrc::RpcError;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sqlx::{Executor, Postgres};
+use tracing::instrument;
+
+use crate::context::{CliContext, RpcContext};
+use crate::middleware::auth::{AsLogoutSessionId, HasLoggedOutSessions, HashSessionToken};
+use crate::middleware::encrypt::EncryptedWire;
+use crate::prelude::*;
+use crate::util::display_none;
+use crate::util::serde::{display_serializable, IoFormat};
+use crate::{ensure_code, Error, ResultExt};
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum PasswordType {
+    EncryptedWire(EncryptedWire),
+    String(String),
+}
+impl PasswordType {
+    pub fn decrypt(self, current_secret: impl AsRef<Jwk>) -> Result<String, Error> {
+        match self {
+            PasswordType::String(x) => Ok(x),
+            PasswordType::EncryptedWire(x) => x.decrypt(current_secret).ok_or_else(|| {
+                Error::new(
+                    color_eyre::eyre::eyre!("Couldn't decode password"),
+                    crate::ErrorKind::Unknown,
+                )
+            }),
+        }
+    }
+}
+impl Default for PasswordType {
+    fn default() -> Self {
+        PasswordType::String(String::default())
+    }
+}
+impl std::fmt::Debug for PasswordType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "<REDACTED_PASSWORD>")?;
+        Ok(())
+    }
+}
+
+impl std::str::FromStr for PasswordType {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match serde_json::from_str(s) {
+            Ok(a) => a,
+            Err(_) => PasswordType::String(s.to_string()),
+        })
+    }
+}
+
+#[command(subcommands(login, logout, session, reset_password, get_pubkey))]
+pub fn auth() -> Result<(), Error> {
+    Ok(())
+}
+
+pub fn cli_metadata() -> Value {
+    serde_json::json!({
+        "platforms": ["cli"],
+    })
+}
+
+pub fn parse_metadata(_: &str, _: &ArgMatches) -> Result<Value, Error> {
+    Ok(cli_metadata())
+}
+
+#[test]
+fn gen_pwd() {
+    println!(
+        "{:?}",
+        argon2::hash_encoded(
+            b"testing1234",
+            &rand::random::<[u8; 16]>()[..],
+            &argon2::Config::rfc9106_low_mem()
+        )
+        .unwrap()
+    )
+}
+
+#[instrument(skip_all)]
+async fn cli_login(
+    ctx: CliContext,
+    password: Option<PasswordType>,
+    metadata: Value,
+) -> Result<(), RpcError> {
+    let password = if let Some(password) = password {
+        password.decrypt(&ctx)?
+    } else {
+        rpassword::prompt_password("Password: ")?
+    };
+
+    rpc_toolkit::command_helpers::call_remote(
+        ctx,
+        "auth.login",
+        serde_json::json!({ "password": password, "metadata": metadata }),
+        PhantomData::<()>,
+    )
+    .await?
+    .result?;
+
+    Ok(())
+}
+
+pub fn check_password(hash: &str, password: &str) -> Result<(), Error> {
+    ensure_code!(
+        argon2::verify_encoded(&hash, password.as_bytes()).map_err(|_| {
+            Error::new(
+                eyre!("Password Incorrect"),
+                crate::ErrorKind::IncorrectPassword,
+            )
+        })?,
+        crate::ErrorKind::IncorrectPassword,
+        "Password Incorrect"
+    );
+    Ok(())
+}
+
+pub async fn check_password_against_db<Ex>(secrets: &mut Ex, password: &str) -> Result<(), Error>
+where
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
+{
+    let pw_hash = sqlx::query!("SELECT password FROM account")
+        .fetch_one(secrets)
+        .await?
+        .password;
+    check_password(&pw_hash, password)?;
+    Ok(())
+}
+
+#[command(
+    custom_cli(cli_login(async, context(CliContext))),
+    display(display_none),
+    metadata(authenticated = false)
+)]
+#[instrument(skip_all)]
+pub async fn login(
+    #[context] ctx: RpcContext,
+    #[request] req: &RequestParts,
+    #[response] res: &mut ResponseParts,
+    #[arg] password: Option<PasswordType>,
+    #[arg(
+        parse(parse_metadata),
+        default = "cli_metadata",
+        help = "RPC Only: This value cannot be overidden from the cli"
+    )]
+    metadata: Value,
+) -> Result<(), Error> {
+    let password = password.unwrap_or_default().decrypt(&ctx)?;
+    let mut handle = ctx.secret_store.acquire().await?;
+    check_password_against_db(handle.as_mut(), &password).await?;
+
+    let hash_token = HashSessionToken::new();
+    let user_agent = req.headers.get("user-agent").and_then(|h| h.to_str().ok());
+    let metadata = serde_json::to_string(&metadata).with_kind(crate::ErrorKind::Database)?;
+    let hash_token_hashed = hash_token.hashed();
+    sqlx::query!(
+        "INSERT INTO session (id, user_agent, metadata) VALUES ($1, $2, $3)",
+        hash_token_hashed,
+        user_agent,
+        metadata,
+    )
+    .execute(handle.as_mut())
+    .await?;
+    res.headers.insert(
+        "set-cookie",
+        hash_token.header_value()?, // Should be impossible, but don't want to panic
+    );
+
+    Ok(())
+}
+
+#[command(display(display_none), metadata(authenticated = false))]
+#[instrument(skip_all)]
+pub async fn logout(
+    #[context] ctx: RpcContext,
+    #[request] req: &RequestParts,
+) -> Result<Option<HasLoggedOutSessions>, Error> {
+    let auth = match HashSessionToken::from_request_parts(req) {
+        Err(_) => return Ok(None),
+        Ok(a) => a,
+    };
+    Ok(Some(HasLoggedOutSessions::new(vec![auth], &ctx).await?))
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Session {
+    logged_in: DateTime<Utc>,
+    last_active: DateTime<Utc>,
+    user_agent: Option<String>,
+    metadata: Value,
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SessionList {
+    current: String,
+    sessions: BTreeMap<String, Session>,
+}
+
+#[command(subcommands(list, kill))]
+pub async fn session() -> Result<(), Error> {
+    Ok(())
+}
+
+fn display_sessions(arg: SessionList, matches: &ArgMatches) {
+    use prettytable::*;
+
+    if matches.is_present("format") {
+        return display_serializable(arg, matches);
+    }
+
+    let mut table = Table::new();
+    table.add_row(row![bc =>
+        "ID",
+        "LOGGED IN",
+        "LAST ACTIVE",
+        "USER AGENT",
+        "METADATA",
+    ]);
+    for (id, session) in arg.sessions {
+        let mut row = row![
+            &id,
+            &format!("{}", session.logged_in),
+            &format!("{}", session.last_active),
+            session.user_agent.as_deref().unwrap_or("N/A"),
+            &format!("{}", session.metadata),
+        ];
+        if id == arg.current {
+            row.iter_mut()
+                .map(|c| c.style(Attr::ForegroundColor(color::GREEN)))
+                .collect::<()>()
+        }
+        table.add_row(row);
+    }
+    table.print_tty(false).unwrap();
+}
+
+#[command(display(display_sessions))]
+#[instrument(skip_all)]
+pub async fn list(
+    #[context] ctx: RpcContext,
+    #[request] req: &RequestParts,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+) -> Result<SessionList, Error> {
+    Ok(SessionList {
+        current: HashSessionToken::from_request_parts(req)?.as_hash(),
+        sessions: sqlx::query!(
+            "SELECT * FROM session WHERE logged_out IS NULL OR logged_out > CURRENT_TIMESTAMP"
+        )
+        .fetch_all(ctx.secret_store.acquire().await?.as_mut())
+        .await?
+        .into_iter()
+        .map(|row| {
+            Ok((
+                row.id,
+                Session {
+                    logged_in: DateTime::from_utc(row.logged_in, Utc),
+                    last_active: DateTime::from_utc(row.last_active, Utc),
+                    user_agent: row.user_agent,
+                    metadata: serde_json::from_str(&row.metadata)
+                        .with_kind(crate::ErrorKind::Database)?,
+                },
+            ))
+        })
+        .collect::<Result<_, Error>>()?,
+    })
+}
+
+fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<String>, RpcError> {
+    Ok(arg.split(",").map(|s| s.trim().to_owned()).collect())
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct KillSessionId(String);
+
+impl AsLogoutSessionId for KillSessionId {
+    fn as_logout_session_id(self) -> String {
+        self.0
+    }
+}
+
+#[command(display(display_none))]
+#[instrument(skip_all)]
+pub async fn kill(
+    #[context] ctx: RpcContext,
+    #[arg(parse(parse_comma_separated))] ids: Vec<String>,
+) -> Result<(), Error> {
+    HasLoggedOutSessions::new(ids.into_iter().map(KillSessionId), &ctx).await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+async fn cli_reset_password(
+    ctx: CliContext,
+    old_password: Option<PasswordType>,
+    new_password: Option<PasswordType>,
+) -> Result<(), RpcError> {
+    let old_password = if let Some(old_password) = old_password {
+        old_password.decrypt(&ctx)?
+    } else {
+        rpassword::prompt_password("Current Password: ")?
+    };
+
+    let new_password = if let Some(new_password) = new_password {
+        new_password.decrypt(&ctx)?
+    } else {
+        let new_password = rpassword::prompt_password("New Password: ")?;
+        if new_password != rpassword::prompt_password("Confirm: ")? {
+            return Err(Error::new(
+                eyre!("Passwords do not match"),
+                crate::ErrorKind::IncorrectPassword,
+            )
+            .into());
+        }
+        new_password
+    };
+
+    rpc_toolkit::command_helpers::call_remote(
+        ctx,
+        "auth.reset-password",
+        serde_json::json!({ "old-password": old_password, "new-password": new_password }),
+        PhantomData::<()>,
+    )
+    .await?
+    .result?;
+
+    Ok(())
+}
+
+#[command(
+    rename = "reset-password",
+    custom_cli(cli_reset_password(async, context(CliContext))),
+    display(display_none)
+)]
+#[instrument(skip_all)]
+pub async fn reset_password(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "old-password")] old_password: Option<PasswordType>,
+    #[arg(rename = "new-password")] new_password: Option<PasswordType>,
+) -> Result<(), Error> {
+    let old_password = old_password.unwrap_or_default().decrypt(&ctx)?;
+    let new_password = new_password.unwrap_or_default().decrypt(&ctx)?;
+
+    let mut account = ctx.account.write().await;
+    if !argon2::verify_encoded(&account.password, old_password.as_bytes())
+        .with_kind(crate::ErrorKind::IncorrectPassword)?
+    {
+        return Err(Error::new(
+            eyre!("Incorrect Password"),
+            crate::ErrorKind::IncorrectPassword,
+        ));
+    }
+    account.set_password(&new_password)?;
+    account.save(&ctx.secret_store).await?;
+    let account_password = &account.password;
+    ctx.db
+        .mutate(|d| {
+            d.as_server_info_mut()
+                .as_password_hash_mut()
+                .ser(account_password)
+        })
+        .await
+}
+
+#[command(
+    rename = "get-pubkey",
+    display(display_none),
+    metadata(authenticated = false)
+)]
+#[instrument(skip_all)]
+pub async fn get_pubkey(#[context] ctx: RpcContext) -> Result<Jwk, RpcError> {
+    let secret = ctx.as_ref().clone();
+    let pub_key = secret.to_public_key()?;
+    Ok(pub_key)
+}

core/startos/src/backup/backup_bulk.rs

@@ -0,0 +1,322 @@
+use std::collections::BTreeMap;
+use std::panic::UnwindSafe;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use chrono::Utc;
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use helpers::AtomicFile;
+use imbl::OrdSet;
+use models::Version;
+use rpc_toolkit::command;
+use tokio::io::AsyncWriteExt;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+use super::target::BackupTargetId;
+use super::PackageBackupReport;
+use crate::auth::check_password_against_db;
+use crate::backup::os::OsBackup;
+use crate::backup::{BackupReport, ServerBackupReport};
+use crate::context::RpcContext;
+use crate::db::model::BackupProgress;
+use crate::db::package::get_packages;
+use crate::disk::mount::backup::BackupMountGuard;
+use crate::disk::mount::filesystem::ReadWrite;
+use crate::disk::mount::guard::TmpMountGuard;
+use crate::manager::BackupReturn;
+use crate::notifications::NotificationLevel;
+use crate::prelude::*;
+use crate::s9pk::manifest::PackageId;
+use crate::util::display_none;
+use crate::util::io::dir_copy;
+use crate::util::serde::IoFormat;
+use crate::version::VersionT;
+
+fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<OrdSet<PackageId>, Error> {
+    arg.split(',')
+        .map(|s| s.trim().parse::<PackageId>().map_err(Error::from))
+        .collect()
+}
+
+#[command(rename = "create", display(display_none))]
+#[instrument(skip(ctx, old_password, password))]
+pub async fn backup_all(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "target-id")] target_id: BackupTargetId,
+    #[arg(rename = "old-password", long = "old-password")] old_password: Option<
+        crate::auth::PasswordType,
+    >,
+    #[arg(
+        rename = "package-ids",
+        long = "package-ids",
+        parse(parse_comma_separated)
+    )]
+    package_ids: Option<OrdSet<PackageId>>,
+    #[arg] password: crate::auth::PasswordType,
+) -> Result<(), Error> {
+    let db = ctx.db.peek().await;
+    let old_password_decrypted = old_password
+        .as_ref()
+        .unwrap_or(&password)
+        .clone()
+        .decrypt(&ctx)?;
+    let password = password.decrypt(&ctx)?;
+    check_password_against_db(ctx.secret_store.acquire().await?.as_mut(), &password).await?;
+    let fs = target_id
+        .load(ctx.secret_store.acquire().await?.as_mut())
+        .await?;
+    let mut backup_guard = BackupMountGuard::mount(
+        TmpMountGuard::mount(&fs, ReadWrite).await?,
+        &old_password_decrypted,
+    )
+    .await?;
+    let package_ids = if let Some(ids) = package_ids {
+        ids.into_iter()
+            .flat_map(|package_id| {
+                let version = db
+                    .as_package_data()
+                    .as_idx(&package_id)?
+                    .as_manifest()
+                    .as_version()
+                    .de()
+                    .ok()?;
+                Some((package_id, version))
+            })
+            .collect()
+    } else {
+        get_packages(db.clone())?.into_iter().collect()
+    };
+    if old_password.is_some() {
+        backup_guard.change_password(&password)?;
+    }
+    assure_backing_up(&ctx.db, &package_ids).await?;
+    tokio::task::spawn(async move {
+        let backup_res = perform_backup(&ctx, backup_guard, &package_ids).await;
+        match backup_res {
+            Ok(report) if report.iter().all(|(_, rep)| rep.error.is_none()) => ctx
+                .notification_manager
+                .notify(
+                    ctx.db.clone(),
+                    None,
+                    NotificationLevel::Success,
+                    "Backup Complete".to_owned(),
+                    "Your backup has completed".to_owned(),
+                    BackupReport {
+                        server: ServerBackupReport {
+                            attempted: true,
+                            error: None,
+                        },
+                        packages: report
+                            .into_iter()
+                            .map(|((package_id, _), value)| (package_id, value))
+                            .collect(),
+                    },
+                    None,
+                )
+                .await
+                .expect("failed to send notification"),
+            Ok(report) => ctx
+                .notification_manager
+                .notify(
+                    ctx.db.clone(),
+                    None,
+                    NotificationLevel::Warning,
+                    "Backup Complete".to_owned(),
+                    "Your backup has completed, but some package(s) failed to backup".to_owned(),
+                    BackupReport {
+                        server: ServerBackupReport {
+                            attempted: true,
+                            error: None,
+                        },
+                        packages: report
+                            .into_iter()
+                            .map(|((package_id, _), value)| (package_id, value))
+                            .collect(),
+                    },
+                    None,
+                )
+                .await
+                .expect("failed to send notification"),
+            Err(e) => {
+                tracing::error!("Backup Failed: {}", e);
+                tracing::debug!("{:?}", e);
+                ctx.notification_manager
+                    .notify(
+                        ctx.db.clone(),
+                        None,
+                        NotificationLevel::Error,
+                        "Backup Failed".to_owned(),
+                        "Your backup failed to complete.".to_owned(),
+                        BackupReport {
+                            server: ServerBackupReport {
+                                attempted: true,
+                                error: Some(e.to_string()),
+                            },
+                            packages: BTreeMap::new(),
+                        },
+                        None,
+                    )
+                    .await
+                    .expect("failed to send notification");
+            }
+        }
+        ctx.db
+            .mutate(|v| {
+                v.as_server_info_mut()
+                    .as_status_info_mut()
+                    .as_backup_progress_mut()
+                    .ser(&None)
+            })
+            .await?;
+        Ok::<(), Error>(())
+    });
+    Ok(())
+}
+
+#[instrument(skip(db, packages))]
+async fn assure_backing_up(
+    db: &PatchDb,
+    packages: impl IntoIterator<Item = &(PackageId, Version)> + UnwindSafe + Send,
+) -> Result<(), Error> {
+    db.mutate(|v| {
+        let backing_up = v
+            .as_server_info_mut()
+            .as_status_info_mut()
+            .as_backup_progress_mut();
+        if backing_up
+            .clone()
+            .de()?
+            .iter()
+            .flat_map(|x| x.values())
+            .fold(false, |acc, x| {
+                if !x.complete {
+                    return true;
+                }
+                acc
+            })
+        {
+            return Err(Error::new(
+                eyre!("Server is already backing up!"),
+                ErrorKind::InvalidRequest,
+            ));
+        }
+        backing_up.ser(&Some(
+            packages
+                .into_iter()
+                .map(|(x, _)| (x.clone(), BackupProgress { complete: false }))
+                .collect(),
+        ))?;
+        Ok(())
+    })
+    .await
+}
+
+#[instrument(skip(ctx, backup_guard))]
+async fn perform_backup(
+    ctx: &RpcContext,
+    backup_guard: BackupMountGuard<TmpMountGuard>,
+    package_ids: &OrdSet<(PackageId, Version)>,
+) -> Result<BTreeMap<(PackageId, Version), PackageBackupReport>, Error> {
+    let mut backup_report = BTreeMap::new();
+    let backup_guard = Arc::new(Mutex::new(backup_guard));
+
+    for package_id in package_ids {
+        let (response, _report) = match ctx
+            .managers
+            .get(package_id)
+            .await
+            .ok_or_else(|| Error::new(eyre!("Manager not found"), ErrorKind::InvalidRequest))?
+            .backup(backup_guard.clone())
+            .await
+        {
+            BackupReturn::Ran { report, res } => (res, report),
+            BackupReturn::AlreadyRunning(report) => {
+                backup_report.insert(package_id.clone(), report);
+                continue;
+            }
+            BackupReturn::Error(error) => {
+                tracing::warn!("Backup thread error");
+                tracing::debug!("{error:?}");
+                backup_report.insert(
+                    package_id.clone(),
+                    PackageBackupReport {
+                        error: Some("Backup thread error".to_owned()),
+                    },
+                );
+                continue;
+            }
+        };
+        backup_report.insert(
+            package_id.clone(),
+            PackageBackupReport {
+                error: response.as_ref().err().map(|e| e.to_string()),
+            },
+        );
+
+        if let Ok(pkg_meta) = response {
+            backup_guard
+                .lock()
+                .await
+                .metadata
+                .package_backups
+                .insert(package_id.0.clone(), pkg_meta);
+        }
+    }
+
+    let ui = ctx.db.peek().await.into_ui().de()?;
+
+    let mut os_backup_file = AtomicFile::new(
+        backup_guard.lock().await.as_ref().join("os-backup.cbor"),
+        None::<PathBuf>,
+    )
+    .await
+    .with_kind(ErrorKind::Filesystem)?;
+    os_backup_file
+        .write_all(&IoFormat::Cbor.to_vec(&OsBackup {
+            account: ctx.account.read().await.clone(),
+            ui,
+        })?)
+        .await?;
+    os_backup_file
+        .save()
+        .await
+        .with_kind(ErrorKind::Filesystem)?;
+
+    let luks_folder_old = backup_guard.lock().await.as_ref().join("luks.old");
+    if tokio::fs::metadata(&luks_folder_old).await.is_ok() {
+        tokio::fs::remove_dir_all(&luks_folder_old).await?;
+    }
+    let luks_folder_bak = backup_guard.lock().await.as_ref().join("luks");
+    if tokio::fs::metadata(&luks_folder_bak).await.is_ok() {
+        tokio::fs::rename(&luks_folder_bak, &luks_folder_old).await?;
+    }
+    let luks_folder = Path::new("/media/embassy/config/luks");
+    if tokio::fs::metadata(&luks_folder).await.is_ok() {
+        dir_copy(&luks_folder, &luks_folder_bak, None).await?;
+    }
+
+    let timestamp = Some(Utc::now());
+    let mut backup_guard = Arc::try_unwrap(backup_guard)
+        .map_err(|_err| {
+            Error::new(
+                eyre!("Backup guard could not ensure that the others where dropped"),
+                ErrorKind::Unknown,
+            )
+        })?
+        .into_inner();
+
+    backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();
+    backup_guard.unencrypted_metadata.full = true;
+    backup_guard.metadata.version = crate::version::Current::new().semver().into();
+    backup_guard.metadata.timestamp = timestamp;
+
+    backup_guard.save_and_unmount().await?;
+
+    ctx.db
+        .mutate(|v| v.as_server_info_mut().as_last_backup_mut().ser(&timestamp))
+        .await?;
+
+    Ok(backup_report)
+}

core/startos/src/backup/mod.rs

@@ -0,0 +1,226 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use chrono::{DateTime, Utc};
+use color_eyre::eyre::eyre;
+use helpers::AtomicFile;
+use models::{ImageId, OptionExt};
+use reqwest::Url;
+use rpc_toolkit::command;
+use serde::{Deserialize, Serialize};
+use tokio::fs::File;
+use tokio::io::AsyncWriteExt;
+use tracing::instrument;
+
+use self::target::PackageBackupInfo;
+use crate::context::RpcContext;
+use crate::install::PKG_ARCHIVE_DIR;
+use crate::manager::manager_seed::ManagerSeed;
+use crate::net::interface::InterfaceId;
+use crate::net::keys::Key;
+use crate::prelude::*;
+use crate::procedure::docker::DockerContainers;
+use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
+use crate::s9pk::manifest::PackageId;
+use crate::util::serde::{Base32, Base64, IoFormat};
+use crate::util::Version;
+use crate::version::{Current, VersionT};
+use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
+use crate::{Error, ErrorKind, ResultExt};
+
+pub mod backup_bulk;
+pub mod os;
+pub mod restore;
+pub mod target;
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct BackupReport {
+    server: ServerBackupReport,
+    packages: BTreeMap<PackageId, PackageBackupReport>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct ServerBackupReport {
+    attempted: bool,
+    error: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct PackageBackupReport {
+    pub error: Option<String>,
+}
+
+#[command(subcommands(backup_bulk::backup_all, target::target))]
+pub fn backup() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(rename = "backup", subcommands(restore::restore_packages_rpc))]
+pub fn package_backup() -> Result<(), Error> {
+    Ok(())
+}
+
+#[derive(Deserialize, Serialize)]
+struct BackupMetadata {
+    pub timestamp: DateTime<Utc>,
+    #[serde(default)]
+    pub network_keys: BTreeMap<InterfaceId, Base64<[u8; 32]>>,
+    #[serde(default)]
+    pub tor_keys: BTreeMap<InterfaceId, Base32<[u8; 64]>>, // DEPRECATED
+    pub marketplace_url: Option<Url>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
+#[model = "Model<Self>"]
+pub struct BackupActions {
+    pub create: PackageProcedure,
+    pub restore: PackageProcedure,
+}
+impl BackupActions {
+    pub fn validate(
+        &self,
+        _container: &Option<DockerContainers>,
+        eos_version: &Version,
+        volumes: &Volumes,
+        image_ids: &BTreeSet<ImageId>,
+    ) -> Result<(), Error> {
+        self.create
+            .validate(eos_version, volumes, image_ids, false)
+            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Create"))?;
+        self.restore
+            .validate(eos_version, volumes, image_ids, false)
+            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Restore"))?;
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn create(&self, seed: Arc<ManagerSeed>) -> Result<PackageBackupInfo, Error> {
+        let manifest = &seed.manifest;
+        let mut volumes = seed.manifest.volumes.to_readonly();
+        let ctx = &seed.ctx;
+        let pkg_id = &manifest.id;
+        let pkg_version = &manifest.version;
+        volumes.insert(VolumeId::Backup, Volume::Backup { readonly: false });
+        let backup_dir = backup_dir(&manifest.id);
+        if tokio::fs::metadata(&backup_dir).await.is_err() {
+            tokio::fs::create_dir_all(&backup_dir).await?
+        }
+        self.create
+            .execute::<(), NoOutput>(
+                ctx,
+                pkg_id,
+                pkg_version,
+                ProcedureName::CreateBackup,
+                &volumes,
+                None,
+                None,
+            )
+            .await?
+            .map_err(|e| eyre!("{}", e.1))
+            .with_kind(crate::ErrorKind::Backup)?;
+        let (network_keys, tor_keys): (Vec<_>, Vec<_>) =
+            Key::for_package(&ctx.secret_store, pkg_id)
+                .await?
+                .into_iter()
+                .filter_map(|k| {
+                    let interface = k.interface().map(|(_, i)| i)?;
+                    Some((
+                        (interface.clone(), Base64(k.as_bytes())),
+                        (interface, Base32(k.tor_key().as_bytes())),
+                    ))
+                })
+                .unzip();
+        let marketplace_url = ctx
+            .db
+            .peek()
+            .await
+            .as_package_data()
+            .as_idx(&pkg_id)
+            .or_not_found(pkg_id)?
+            .expect_as_installed()?
+            .as_installed()
+            .as_marketplace_url()
+            .de()?;
+        let tmp_path = Path::new(BACKUP_DIR)
+            .join(pkg_id)
+            .join(format!("{}.s9pk", pkg_id));
+        let s9pk_path = ctx
+            .datadir
+            .join(PKG_ARCHIVE_DIR)
+            .join(pkg_id)
+            .join(pkg_version.as_str())
+            .join(format!("{}.s9pk", pkg_id));
+        let mut infile = File::open(&s9pk_path).await?;
+        let mut outfile = AtomicFile::new(&tmp_path, None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
+        tokio::io::copy(&mut infile, &mut *outfile)
+            .await
+            .with_ctx(|_| {
+                (
+                    crate::ErrorKind::Filesystem,
+                    format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()),
+                )
+            })?;
+        outfile.save().await.with_kind(ErrorKind::Filesystem)?;
+        let timestamp = Utc::now();
+        let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
+        let mut outfile = AtomicFile::new(&metadata_path, None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
+        let network_keys = network_keys.into_iter().collect();
+        let tor_keys = tor_keys.into_iter().collect();
+        outfile
+            .write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
+                timestamp,
+                network_keys,
+                tor_keys,
+                marketplace_url,
+            })?)
+            .await?;
+        outfile.save().await.with_kind(ErrorKind::Filesystem)?;
+        Ok(PackageBackupInfo {
+            os_version: Current::new().semver().into(),
+            title: manifest.title.clone(),
+            version: pkg_version.clone(),
+            timestamp,
+        })
+    }
+
+    #[instrument(skip_all)]
+    pub async fn restore(
+        &self,
+        ctx: &RpcContext,
+        pkg_id: &PackageId,
+        pkg_version: &Version,
+        volumes: &Volumes,
+    ) -> Result<Option<Url>, Error> {
+        let mut volumes = volumes.clone();
+        volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true });
+        self.restore
+            .execute::<(), NoOutput>(
+                ctx,
+                pkg_id,
+                pkg_version,
+                ProcedureName::RestoreBackup,
+                &volumes,
+                None,
+                None,
+            )
+            .await?
+            .map_err(|e| eyre!("{}", e.1))
+            .with_kind(crate::ErrorKind::Restore)?;
+        let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
+        let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
+            &tokio::fs::read(&metadata_path).await.with_ctx(|_| {
+                (
+                    crate::ErrorKind::Filesystem,
+                    metadata_path.display().to_string(),
+                )
+            })?,
+        )?;
+
+        Ok(metadata.marketplace_url)
+    }
+}

core/startos/src/backup/os.rs

@@ -0,0 +1,122 @@
+use openssl::pkey::PKey;
+use openssl::x509::X509;
+use patch_db::Value;
+use serde::{Deserialize, Serialize};
+
+use crate::account::AccountInfo;
+use crate::hostname::{generate_hostname, generate_id, Hostname};
+use crate::net::keys::Key;
+use crate::prelude::*;
+use crate::util::serde::Base64;
+
+pub struct OsBackup {
+    pub account: AccountInfo,
+    pub ui: Value,
+}
+impl<'de> Deserialize<'de> for OsBackup {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let tagged = OsBackupSerDe::deserialize(deserializer)?;
+        match tagged.version {
+            0 => patch_db::value::from_value::<OsBackupV0>(tagged.rest)
+                .map_err(serde::de::Error::custom)?
+                .project()
+                .map_err(serde::de::Error::custom),
+            1 => patch_db::value::from_value::<OsBackupV1>(tagged.rest)
+                .map_err(serde::de::Error::custom)?
+                .project()
+                .map_err(serde::de::Error::custom),
+            v => Err(serde::de::Error::custom(&format!(
+                "Unknown backup version {v}"
+            ))),
+        }
+    }
+}
+impl Serialize for OsBackup {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        OsBackupSerDe {
+            version: 1,
+            rest: patch_db::value::to_value(
+                &OsBackupV1::unproject(self).map_err(serde::ser::Error::custom)?,
+            )
+            .map_err(serde::ser::Error::custom)?,
+        }
+        .serialize(serializer)
+    }
+}
+
+#[derive(Deserialize, Serialize)]
+struct OsBackupSerDe {
+    #[serde(default)]
+    version: usize,
+    #[serde(flatten)]
+    rest: Value,
+}
+
+/// V0
+#[derive(Deserialize)]
+#[serde(rename = "kebab-case")]
+struct OsBackupV0 {
+    // tor_key: Base32<[u8; 64]>,
+    root_ca_key: String,  // PEM Encoded OpenSSL Key
+    root_ca_cert: String, // PEM Encoded OpenSSL X509 Certificate
+    ui: Value,            // JSON Value
+}
+impl OsBackupV0 {
+    fn project(self) -> Result<OsBackup, Error> {
+        Ok(OsBackup {
+            account: AccountInfo {
+                server_id: generate_id(),
+                hostname: generate_hostname(),
+                password: Default::default(),
+                key: Key::new(None),
+                root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
+                root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
+            },
+            ui: self.ui,
+        })
+    }
+}
+
+/// V1
+#[derive(Deserialize, Serialize)]
+#[serde(rename = "kebab-case")]
+struct OsBackupV1 {
+    server_id: String,         // uuidv4
+    hostname: String,          // embassy-<adjective>-<noun>
+    net_key: Base64<[u8; 32]>, // Ed25519 Secret Key
+    root_ca_key: String,       // PEM Encoded OpenSSL Key
+    root_ca_cert: String,      // PEM Encoded OpenSSL X509 Certificate
+    ui: Value,                 // JSON Value
+                               // TODO add more
+}
+impl OsBackupV1 {
+    fn project(self) -> Result<OsBackup, Error> {
+        Ok(OsBackup {
+            account: AccountInfo {
+                server_id: self.server_id,
+                hostname: Hostname(self.hostname),
+                password: Default::default(),
+                key: Key::from_bytes(None, self.net_key.0),
+                root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
+                root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
+            },
+            ui: self.ui,
+        })
+    }
+    fn unproject(backup: &OsBackup) -> Result<Self, Error> {
+        Ok(Self {
+            server_id: backup.account.server_id.clone(),
+            hostname: backup.account.hostname.0.clone(),
+            net_key: Base64(backup.account.key.as_bytes()),
+            root_ca_key: String::from_utf8(backup.account.root_ca_key.private_key_to_pem_pkcs8()?)?,
+            root_ca_cert: String::from_utf8(backup.account.root_ca_cert.to_pem()?)?,
+            ui: backup.ui.clone(),
+        })
+    }
+}

core/startos/src/backup/restore.rs

@@ -0,0 +1,461 @@
+use std::collections::BTreeMap;
+use std::path::Path;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+use std::time::Duration;
+
+use clap::ArgMatches;
+use futures::future::BoxFuture;
+use futures::{stream, FutureExt, StreamExt};
+use openssl::x509::X509;
+use rpc_toolkit::command;
+use sqlx::Connection;
+use tokio::fs::File;
+use torut::onion::OnionAddressV3;
+use tracing::instrument;
+
+use super::target::BackupTargetId;
+use crate::backup::os::OsBackup;
+use crate::backup::BackupMetadata;
+use crate::context::rpc::RpcContextConfig;
+use crate::context::{RpcContext, SetupContext};
+use crate::db::model::{PackageDataEntry, PackageDataEntryRestoring, StaticFiles};
+use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
+use crate::disk::mount::filesystem::ReadWrite;
+use crate::disk::mount::guard::TmpMountGuard;
+use crate::hostname::Hostname;
+use crate::init::init;
+use crate::install::progress::InstallProgress;
+use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
+use crate::notifications::NotificationLevel;
+use crate::prelude::*;
+use crate::s9pk::manifest::{Manifest, PackageId};
+use crate::s9pk::reader::S9pkReader;
+use crate::setup::SetupStatus;
+use crate::util::display_none;
+use crate::util::io::dir_size;
+use crate::util::serde::IoFormat;
+use crate::volume::{backup_dir, BACKUP_DIR, PKG_VOLUME_DIR};
+
+fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<PackageId>, Error> {
+    arg.split(',')
+        .map(|s| s.trim().parse().map_err(Error::from))
+        .collect()
+}
+
+#[command(rename = "restore", display(display_none))]
+#[instrument(skip(ctx, password))]
+pub async fn restore_packages_rpc(
+    #[context] ctx: RpcContext,
+    #[arg(parse(parse_comma_separated))] ids: Vec<PackageId>,
+    #[arg(rename = "target-id")] target_id: BackupTargetId,
+    #[arg] password: String,
+) -> Result<(), Error> {
+    let fs = target_id
+        .load(ctx.secret_store.acquire().await?.as_mut())
+        .await?;
+    let backup_guard =
+        BackupMountGuard::mount(TmpMountGuard::mount(&fs, ReadWrite).await?, &password).await?;
+
+    let (backup_guard, tasks, _) = restore_packages(&ctx, backup_guard, ids).await?;
+
+    tokio::spawn(async move {
+        stream::iter(tasks.into_iter().map(|x| (x, ctx.clone())))
+            .for_each_concurrent(5, |(res, ctx)| async move {
+                match res.await {
+                    (Ok(_), _) => (),
+                    (Err(err), package_id) => {
+                        if let Err(err) = ctx
+                            .notification_manager
+                            .notify(
+                                ctx.db.clone(),
+                                Some(package_id.clone()),
+                                NotificationLevel::Error,
+                                "Restoration Failure".to_string(),
+                                format!("Error restoring package {}: {}", package_id, err),
+                                (),
+                                None,
+                            )
+                            .await
+                        {
+                            tracing::error!("Failed to notify: {}", err);
+                            tracing::debug!("{:?}", err);
+                        };
+                        tracing::error!("Error restoring package {}: {}", package_id, err);
+                        tracing::debug!("{:?}", err);
+                    }
+                }
+            })
+            .await;
+        if let Err(e) = backup_guard.unmount().await {
+            tracing::error!("Error unmounting backup drive: {}", e);
+            tracing::debug!("{:?}", e);
+        }
+    });
+
+    Ok(())
+}
+
+async fn approximate_progress(
+    rpc_ctx: &RpcContext,
+    progress: &mut ProgressInfo,
+) -> Result<(), Error> {
+    for (id, size) in &mut progress.target_volume_size {
+        let dir = rpc_ctx.datadir.join(PKG_VOLUME_DIR).join(id).join("data");
+        if tokio::fs::metadata(&dir).await.is_err() {
+            *size = 0;
+        } else {
+            *size = dir_size(&dir, None).await?;
+        }
+    }
+    Ok(())
+}
+
+async fn approximate_progress_loop(
+    ctx: &SetupContext,
+    rpc_ctx: &RpcContext,
+    mut starting_info: ProgressInfo,
+) {
+    loop {
+        if let Err(e) = approximate_progress(rpc_ctx, &mut starting_info).await {
+            tracing::error!("Failed to approximate restore progress: {}", e);
+            tracing::debug!("{:?}", e);
+        } else {
+            *ctx.setup_status.write().await = Some(Ok(starting_info.flatten()));
+        }
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+}
+
+#[derive(Debug, Default)]
+struct ProgressInfo {
+    package_installs: BTreeMap<PackageId, Arc<InstallProgress>>,
+    src_volume_size: BTreeMap<PackageId, u64>,
+    target_volume_size: BTreeMap<PackageId, u64>,
+}
+impl ProgressInfo {
+    fn flatten(&self) -> SetupStatus {
+        let mut total_bytes = 0;
+        let mut bytes_transferred = 0;
+
+        for progress in self.package_installs.values() {
+            total_bytes += ((progress.size.unwrap_or(0) as f64) * 2.2) as u64;
+            bytes_transferred += progress.downloaded.load(Ordering::SeqCst);
+            bytes_transferred += ((progress.validated.load(Ordering::SeqCst) as f64) * 0.2) as u64;
+            bytes_transferred += progress.unpacked.load(Ordering::SeqCst);
+        }
+
+        for size in self.src_volume_size.values() {
+            total_bytes += *size;
+        }
+
+        for size in self.target_volume_size.values() {
+            bytes_transferred += *size;
+        }
+
+        if bytes_transferred > total_bytes {
+            bytes_transferred = total_bytes;
+        }
+
+        SetupStatus {
+            total_bytes: Some(total_bytes),
+            bytes_transferred,
+            complete: false,
+        }
+    }
+}
+
+#[instrument(skip(ctx))]
+pub async fn recover_full_embassy(
+    ctx: SetupContext,
+    disk_guid: Arc<String>,
+    embassy_password: String,
+    recovery_source: TmpMountGuard,
+    recovery_password: Option<String>,
+) -> Result<(Arc<String>, Hostname, OnionAddressV3, X509), Error> {
+    let backup_guard = BackupMountGuard::mount(
+        recovery_source,
+        recovery_password.as_deref().unwrap_or_default(),
+    )
+    .await?;
+
+    let os_backup_path = backup_guard.as_ref().join("os-backup.cbor");
+    let mut os_backup: OsBackup = IoFormat::Cbor.from_slice(
+        &tokio::fs::read(&os_backup_path)
+            .await
+            .with_ctx(|_| (ErrorKind::Filesystem, os_backup_path.display().to_string()))?,
+    )?;
+
+    os_backup.account.password = argon2::hash_encoded(
+        embassy_password.as_bytes(),
+        &rand::random::<[u8; 16]>()[..],
+        &argon2::Config::rfc9106_low_mem(),
+    )
+    .with_kind(ErrorKind::PasswordHashGeneration)?;
+
+    let secret_store = ctx.secret_store().await?;
+
+    os_backup.account.save(&secret_store).await?;
+
+    secret_store.close().await;
+
+    let cfg = RpcContextConfig::load(ctx.config_path.clone()).await?;
+
+    init(&cfg).await?;
+
+    let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid.clone()).await?;
+
+    let ids: Vec<_> = backup_guard
+        .metadata
+        .package_backups
+        .keys()
+        .cloned()
+        .collect();
+    let (backup_guard, tasks, progress_info) =
+        restore_packages(&rpc_ctx, backup_guard, ids).await?;
+    let task_consumer_rpc_ctx = rpc_ctx.clone();
+    tokio::select! {
+        _ = async move {
+            stream::iter(tasks.into_iter().map(|x| (x,  task_consumer_rpc_ctx.clone())))
+                .for_each_concurrent(5, |(res, ctx)| async move {
+                    match res.await {
+                        (Ok(_), _) => (),
+                        (Err(err), package_id) => {
+                            if let Err(err) = ctx.notification_manager.notify(
+                                ctx.db.clone(),
+                                Some(package_id.clone()),
+                                NotificationLevel::Error,
+                                "Restoration Failure".to_string(), format!("Error restoring package {}: {}", package_id,err), (), None).await{
+                                tracing::error!("Failed to notify: {}", err);
+                                tracing::debug!("{:?}", err);
+                                };
+                            tracing::error!("Error restoring package {}: {}", package_id, err);
+                            tracing::debug!("{:?}", err);
+                        },
+                    }
+                }).await;
+
+        } => {
+
+        },
+        _ = approximate_progress_loop(&ctx, &rpc_ctx, progress_info) => unreachable!(concat!(module_path!(), "::approximate_progress_loop should not terminate")),
+    }
+
+    backup_guard.unmount().await?;
+    rpc_ctx.shutdown().await?;
+
+    Ok((
+        disk_guid,
+        os_backup.account.hostname,
+        os_backup.account.key.tor_address(),
+        os_backup.account.root_ca_cert,
+    ))
+}
+
+#[instrument(skip(ctx, backup_guard))]
+async fn restore_packages(
+    ctx: &RpcContext,
+    backup_guard: BackupMountGuard<TmpMountGuard>,
+    ids: Vec<PackageId>,
+) -> Result<
+    (
+        BackupMountGuard<TmpMountGuard>,
+        Vec<BoxFuture<'static, (Result<(), Error>, PackageId)>>,
+        ProgressInfo,
+    ),
+    Error,
+> {
+    let guards = assure_restoring(ctx, ids, &backup_guard).await?;
+
+    let mut progress_info = ProgressInfo::default();
+
+    let mut tasks = Vec::with_capacity(guards.len());
+    for (manifest, guard) in guards {
+        let id = manifest.id.clone();
+        let (progress, task) = restore_package(ctx.clone(), manifest, guard).await?;
+        progress_info
+            .package_installs
+            .insert(id.clone(), progress.clone());
+        progress_info
+            .src_volume_size
+            .insert(id.clone(), dir_size(backup_dir(&id), None).await?);
+        progress_info.target_volume_size.insert(id.clone(), 0);
+        let package_id = id.clone();
+        tasks.push(
+            async move {
+                if let Err(e) = task.await {
+                    tracing::error!("Error restoring package {}: {}", id, e);
+                    tracing::debug!("{:?}", e);
+                    Err(e)
+                } else {
+                    Ok(())
+                }
+            }
+            .map(|x| (x, package_id))
+            .boxed(),
+        );
+    }
+
+    Ok((backup_guard, tasks, progress_info))
+}
+
+#[instrument(skip(ctx, backup_guard))]
+async fn assure_restoring(
+    ctx: &RpcContext,
+    ids: Vec<PackageId>,
+    backup_guard: &BackupMountGuard<TmpMountGuard>,
+) -> Result<Vec<(Manifest, PackageBackupMountGuard)>, Error> {
+    let mut guards = Vec::with_capacity(ids.len());
+
+    let mut insert_packages = BTreeMap::new();
+
+    for id in ids {
+        let peek = ctx.db.peek().await;
+
+        let model = peek.as_package_data().as_idx(&id);
+
+        if !model.is_none() {
+            return Err(Error::new(
+                eyre!("Can't restore over existing package: {}", id),
+                crate::ErrorKind::InvalidRequest,
+            ));
+        }
+        let guard = backup_guard.mount_package_backup(&id).await?;
+        let s9pk_path = Path::new(BACKUP_DIR).join(&id).join(format!("{}.s9pk", id));
+        let mut rdr = S9pkReader::open(&s9pk_path, false).await?;
+
+        let manifest = rdr.manifest().await?;
+        let version = manifest.version.clone();
+        let progress = Arc::new(InstallProgress::new(Some(
+            tokio::fs::metadata(&s9pk_path).await?.len(),
+        )));
+
+        let public_dir_path = ctx
+            .datadir
+            .join(PKG_PUBLIC_DIR)
+            .join(&id)
+            .join(version.as_str());
+        tokio::fs::create_dir_all(&public_dir_path).await?;
+
+        let license_path = public_dir_path.join("LICENSE.md");
+        let mut dst = File::create(&license_path).await?;
+        tokio::io::copy(&mut rdr.license().await?, &mut dst).await?;
+        dst.sync_all().await?;
+
+        let instructions_path = public_dir_path.join("INSTRUCTIONS.md");
+        let mut dst = File::create(&instructions_path).await?;
+        tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?;
+        dst.sync_all().await?;
+
+        let icon_path = Path::new("icon").with_extension(&manifest.assets.icon_type());
+        let icon_path = public_dir_path.join(&icon_path);
+        let mut dst = File::create(&icon_path).await?;
+        tokio::io::copy(&mut rdr.icon().await?, &mut dst).await?;
+        dst.sync_all().await?;
+        insert_packages.insert(
+            id.clone(),
+            PackageDataEntry::Restoring(PackageDataEntryRestoring {
+                install_progress: progress.clone(),
+                static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()),
+                manifest: manifest.clone(),
+            }),
+        );
+
+        guards.push((manifest, guard));
+    }
+    ctx.db
+        .mutate(|db| {
+            for (id, package) in insert_packages {
+                db.as_package_data_mut().insert(&id, &package)?;
+            }
+            Ok(())
+        })
+        .await?;
+    Ok(guards)
+}
+
+#[instrument(skip(ctx, guard))]
+async fn restore_package<'a>(
+    ctx: RpcContext,
+    manifest: Manifest,
+    guard: PackageBackupMountGuard,
+) -> Result<(Arc<InstallProgress>, BoxFuture<'static, Result<(), Error>>), Error> {
+    let id = manifest.id.clone();
+    let s9pk_path = Path::new(BACKUP_DIR)
+        .join(&manifest.id)
+        .join(format!("{}.s9pk", id));
+
+    let metadata_path = Path::new(BACKUP_DIR).join(&id).join("metadata.cbor");
+    let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
+        &tokio::fs::read(&metadata_path)
+            .await
+            .with_ctx(|_| (ErrorKind::Filesystem, metadata_path.display().to_string()))?,
+    )?;
+
+    let mut secrets = ctx.secret_store.acquire().await?;
+    let mut secrets_tx = secrets.begin().await?;
+    for (iface, key) in metadata.network_keys {
+        let k = key.0.as_slice();
+        sqlx::query!(
+            "INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
+            id.to_string(),
+            iface.to_string(),
+            k,
+        )
+        .execute(secrets_tx.as_mut()).await?;
+    }
+    // DEPRECATED
+    for (iface, key) in metadata.tor_keys {
+        let k = key.0.as_slice();
+        sqlx::query!(
+            "INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
+            id.to_string(),
+            iface.to_string(),
+            k,
+        )
+        .execute(secrets_tx.as_mut()).await?;
+    }
+    secrets_tx.commit().await?;
+    drop(secrets);
+
+    let len = tokio::fs::metadata(&s9pk_path)
+        .await
+        .with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?
+        .len();
+    let file = File::open(&s9pk_path)
+        .await
+        .with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?;
+
+    let progress = InstallProgress::new(Some(len));
+    let marketplace_url = metadata.marketplace_url;
+
+    let progress = Arc::new(progress);
+
+    ctx.db
+        .mutate(|db| {
+            db.as_package_data_mut().insert(
+                &id,
+                &PackageDataEntry::Restoring(PackageDataEntryRestoring {
+                    install_progress: progress.clone(),
+                    static_files: StaticFiles::local(
+                        &id,
+                        &manifest.version,
+                        manifest.assets.icon_type(),
+                    ),
+                    manifest: manifest.clone(),
+                }),
+            )
+        })
+        .await?;
+    Ok((
+        progress.clone(),
+        async move {
+            download_install_s9pk(ctx, manifest, marketplace_url, progress, file, None).await?;
+
+            guard.unmount().await?;
+
+            Ok(())
+        }
+        .boxed(),
+    ))
+}

core/startos/src/backup/target/cifs.rs

@@ -0,0 +1,211 @@
+use std::path::{Path, PathBuf};
+
+use color_eyre::eyre::eyre;
+use futures::TryStreamExt;
+use rpc_toolkit::command;
+use serde::{Deserialize, Serialize};
+use sqlx::{Executor, Postgres};
+
+use super::{BackupTarget, BackupTargetId};
+use crate::context::RpcContext;
+use crate::disk::mount::filesystem::cifs::Cifs;
+use crate::disk::mount::filesystem::ReadOnly;
+use crate::disk::mount::guard::TmpMountGuard;
+use crate::disk::util::{recovery_info, EmbassyOsRecoveryInfo};
+use crate::prelude::*;
+use crate::util::display_none;
+use crate::util::serde::KeyVal;
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct CifsBackupTarget {
+    hostname: String,
+    path: PathBuf,
+    username: String,
+    mountable: bool,
+    embassy_os: Option<EmbassyOsRecoveryInfo>,
+}
+
+#[command(subcommands(add, update, remove))]
+pub fn cifs() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(display(display_none))]
+pub async fn add(
+    #[context] ctx: RpcContext,
+    #[arg] hostname: String,
+    #[arg] path: PathBuf,
+    #[arg] username: String,
+    #[arg] password: Option<String>,
+) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
+    let cifs = Cifs {
+        hostname,
+        path,
+        username,
+        password,
+    };
+    let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
+    let embassy_os = recovery_info(&guard).await?;
+    guard.unmount().await?;
+    let path_string = Path::new("/").join(&cifs.path).display().to_string();
+    let id: i32 = sqlx::query!(
+        "INSERT INTO cifs_shares (hostname, path, username, password) VALUES ($1, $2, $3, $4) RETURNING id",
+        cifs.hostname,
+        path_string,
+        cifs.username,
+        cifs.password,
+    )
+    .fetch_one(&ctx.secret_store)
+    .await?.id;
+    Ok(KeyVal {
+        key: BackupTargetId::Cifs { id },
+        value: BackupTarget::Cifs(CifsBackupTarget {
+            hostname: cifs.hostname,
+            path: cifs.path,
+            username: cifs.username,
+            mountable: true,
+            embassy_os,
+        }),
+    })
+}
+
+#[command(display(display_none))]
+pub async fn update(
+    #[context] ctx: RpcContext,
+    #[arg] id: BackupTargetId,
+    #[arg] hostname: String,
+    #[arg] path: PathBuf,
+    #[arg] username: String,
+    #[arg] password: Option<String>,
+) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
+    let id = if let BackupTargetId::Cifs { id } = id {
+        id
+    } else {
+        return Err(Error::new(
+            eyre!("Backup Target ID {} Not Found", id),
+            ErrorKind::NotFound,
+        ));
+    };
+    let cifs = Cifs {
+        hostname,
+        path,
+        username,
+        password,
+    };
+    let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
+    let embassy_os = recovery_info(&guard).await?;
+    guard.unmount().await?;
+    let path_string = Path::new("/").join(&cifs.path).display().to_string();
+    if sqlx::query!(
+        "UPDATE cifs_shares SET hostname = $1, path = $2, username = $3, password = $4 WHERE id = $5",
+        cifs.hostname,
+        path_string,
+        cifs.username,
+        cifs.password,
+        id,
+    )
+    .execute(&ctx.secret_store)
+    .await?
+    .rows_affected()
+        == 0
+    {
+        return Err(Error::new(
+            eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
+            ErrorKind::NotFound,
+        ));
+    };
+    Ok(KeyVal {
+        key: BackupTargetId::Cifs { id },
+        value: BackupTarget::Cifs(CifsBackupTarget {
+            hostname: cifs.hostname,
+            path: cifs.path,
+            username: cifs.username,
+            mountable: true,
+            embassy_os,
+        }),
+    })
+}
+
+#[command(display(display_none))]
+pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Result<(), Error> {
+    let id = if let BackupTargetId::Cifs { id } = id {
+        id
+    } else {
+        return Err(Error::new(
+            eyre!("Backup Target ID {} Not Found", id),
+            ErrorKind::NotFound,
+        ));
+    };
+    if sqlx::query!("DELETE FROM cifs_shares WHERE id = $1", id)
+        .execute(&ctx.secret_store)
+        .await?
+        .rows_affected()
+        == 0
+    {
+        return Err(Error::new(
+            eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
+            ErrorKind::NotFound,
+        ));
+    };
+    Ok(())
+}
+
+pub async fn load<Ex>(secrets: &mut Ex, id: i32) -> Result<Cifs, Error>
+where
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
+{
+    let record = sqlx::query!(
+        "SELECT hostname, path, username, password FROM cifs_shares WHERE id = $1",
+        id
+    )
+    .fetch_one(secrets)
+    .await?;
+
+    Ok(Cifs {
+        hostname: record.hostname,
+        path: PathBuf::from(record.path),
+        username: record.username,
+        password: record.password,
+    })
+}
+
+pub async fn list<Ex>(secrets: &mut Ex) -> Result<Vec<(i32, CifsBackupTarget)>, Error>
+where
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
+{
+    let mut records =
+        sqlx::query!("SELECT id, hostname, path, username, password FROM cifs_shares")
+            .fetch_many(secrets);
+
+    let mut cifs = Vec::new();
+    while let Some(query_result) = records.try_next().await? {
+        if let Some(record) = query_result.right() {
+            let mount_info = Cifs {
+                hostname: record.hostname,
+                path: PathBuf::from(record.path),
+                username: record.username,
+                password: record.password,
+            };
+            let embassy_os = async {
+                let guard = TmpMountGuard::mount(&mount_info, ReadOnly).await?;
+                let embassy_os = recovery_info(&guard).await?;
+                guard.unmount().await?;
+                Ok::<_, Error>(embassy_os)
+            }
+            .await;
+            cifs.push((
+                record.id,
+                CifsBackupTarget {
+                    hostname: mount_info.hostname,
+                    path: mount_info.path,
+                    username: mount_info.username,
+                    mountable: embassy_os.is_ok(),
+                    embassy_os: embassy_os.ok().and_then(|a| a),
+                },
+            ));
+        }
+    }
+
+    Ok(cifs)
+}

core/startos/src/backup/target/mod.rs

@@ -0,0 +1,307 @@
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use digest::generic_array::GenericArray;
+use digest::OutputSizeUser;
+use rpc_toolkit::command;
+use serde::{Deserialize, Serialize};
+use sha2::Sha256;
+use sqlx::{Executor, Postgres};
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+use self::cifs::CifsBackupTarget;
+use crate::context::RpcContext;
+use crate::disk::mount::backup::BackupMountGuard;
+use crate::disk::mount::filesystem::block_dev::BlockDev;
+use crate::disk::mount::filesystem::cifs::Cifs;
+use crate::disk::mount::filesystem::{FileSystem, MountType, ReadWrite};
+use crate::disk::mount::guard::TmpMountGuard;
+use crate::disk::util::PartitionInfo;
+use crate::prelude::*;
+use crate::s9pk::manifest::PackageId;
+use crate::util::serde::{deserialize_from_str, display_serializable, serialize_display};
+use crate::util::{display_none, Version};
+
+pub mod cifs;
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(tag = "type")]
+#[serde(rename_all = "kebab-case")]
+pub enum BackupTarget {
+    #[serde(rename_all = "kebab-case")]
+    Disk {
+        vendor: Option<String>,
+        model: Option<String>,
+        #[serde(flatten)]
+        partition_info: PartitionInfo,
+    },
+    Cifs(CifsBackupTarget),
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
+pub enum BackupTargetId {
+    Disk { logicalname: PathBuf },
+    Cifs { id: i32 },
+}
+impl BackupTargetId {
+    pub async fn load<Ex>(self, secrets: &mut Ex) -> Result<BackupTargetFS, Error>
+    where
+        for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
+    {
+        Ok(match self {
+            BackupTargetId::Disk { logicalname } => {
+                BackupTargetFS::Disk(BlockDev::new(logicalname))
+            }
+            BackupTargetId::Cifs { id } => BackupTargetFS::Cifs(cifs::load(secrets, id).await?),
+        })
+    }
+}
+impl std::fmt::Display for BackupTargetId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BackupTargetId::Disk { logicalname } => write!(f, "disk-{}", logicalname.display()),
+            BackupTargetId::Cifs { id } => write!(f, "cifs-{}", id),
+        }
+    }
+}
+impl std::str::FromStr for BackupTargetId {
+    type Err = Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.split_once('-') {
+            Some(("disk", logicalname)) => Ok(BackupTargetId::Disk {
+                logicalname: Path::new(logicalname).to_owned(),
+            }),
+            Some(("cifs", id)) => Ok(BackupTargetId::Cifs { id: id.parse()? }),
+            _ => Err(Error::new(
+                eyre!("Invalid Backup Target ID"),
+                ErrorKind::InvalidBackupTargetId,
+            )),
+        }
+    }
+}
+impl<'de> Deserialize<'de> for BackupTargetId {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        deserialize_from_str(deserializer)
+    }
+}
+impl Serialize for BackupTargetId {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serialize_display(self, serializer)
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(tag = "type")]
+#[serde(rename_all = "kebab-case")]
+pub enum BackupTargetFS {
+    Disk(BlockDev<PathBuf>),
+    Cifs(Cifs),
+}
+#[async_trait]
+impl FileSystem for BackupTargetFS {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        match self {
+            BackupTargetFS::Disk(a) => a.mount(mountpoint, mount_type).await,
+            BackupTargetFS::Cifs(a) => a.mount(mountpoint, mount_type).await,
+        }
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        match self {
+            BackupTargetFS::Disk(a) => a.source_hash().await,
+            BackupTargetFS::Cifs(a) => a.source_hash().await,
+        }
+    }
+}
+
+#[command(subcommands(cifs::cifs, list, info, mount, umount))]
+pub fn target() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(display(display_serializable))]
+pub async fn list(
+    #[context] ctx: RpcContext,
+) -> Result<BTreeMap<BackupTargetId, BackupTarget>, Error> {
+    let mut sql_handle = ctx.secret_store.acquire().await?;
+    let (disks_res, cifs) = tokio::try_join!(
+        crate::disk::util::list(&ctx.os_partitions),
+        cifs::list(sql_handle.as_mut()),
+    )?;
+    Ok(disks_res
+        .into_iter()
+        .flat_map(|mut disk| {
+            std::mem::take(&mut disk.partitions)
+                .into_iter()
+                .map(|part| {
+                    (
+                        BackupTargetId::Disk {
+                            logicalname: part.logicalname.clone(),
+                        },
+                        BackupTarget::Disk {
+                            vendor: disk.vendor.clone(),
+                            model: disk.model.clone(),
+                            partition_info: part,
+                        },
+                    )
+                })
+                .collect::<Vec<_>>()
+        })
+        .chain(
+            cifs.into_iter()
+                .map(|(id, cifs)| (BackupTargetId::Cifs { id }, BackupTarget::Cifs(cifs))),
+        )
+        .collect())
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct BackupInfo {
+    pub version: Version,
+    pub timestamp: Option<DateTime<Utc>>,
+    pub package_backups: BTreeMap<PackageId, PackageBackupInfo>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct PackageBackupInfo {
+    pub title: String,
+    pub version: Version,
+    pub os_version: Version,
+    pub timestamp: DateTime<Utc>,
+}
+
+fn display_backup_info(info: BackupInfo, matches: &ArgMatches) {
+    use prettytable::*;
+
+    if matches.is_present("format") {
+        return display_serializable(info, matches);
+    }
+
+    let mut table = Table::new();
+    table.add_row(row![bc =>
+        "ID",
+        "VERSION",
+        "OS VERSION",
+        "TIMESTAMP",
+    ]);
+    table.add_row(row![
+        "EMBASSY OS",
+        info.version.as_str(),
+        info.version.as_str(),
+        &if let Some(ts) = &info.timestamp {
+            ts.to_string()
+        } else {
+            "N/A".to_owned()
+        },
+    ]);
+    for (id, info) in info.package_backups {
+        let row = row![
+            &*id,
+            info.version.as_str(),
+            info.os_version.as_str(),
+            &info.timestamp.to_string(),
+        ];
+        table.add_row(row);
+    }
+    table.print_tty(false).unwrap();
+}
+
+#[command(display(display_backup_info))]
+#[instrument(skip(ctx, password))]
+pub async fn info(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "target-id")] target_id: BackupTargetId,
+    #[arg] password: String,
+) -> Result<BackupInfo, Error> {
+    let guard = BackupMountGuard::mount(
+        TmpMountGuard::mount(
+            &target_id
+                .load(ctx.secret_store.acquire().await?.as_mut())
+                .await?,
+            ReadWrite,
+        )
+        .await?,
+        &password,
+    )
+    .await?;
+
+    let res = guard.metadata.clone();
+
+    guard.unmount().await?;
+
+    Ok(res)
+}
+
+lazy_static::lazy_static! {
+    static ref USER_MOUNTS: Mutex<BTreeMap<BackupTargetId, BackupMountGuard<TmpMountGuard>>> =
+        Mutex::new(BTreeMap::new());
+}
+
+#[command]
+#[instrument(skip_all)]
+pub async fn mount(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "target-id")] target_id: BackupTargetId,
+    #[arg] password: String,
+) -> Result<String, Error> {
+    let mut mounts = USER_MOUNTS.lock().await;
+
+    if let Some(existing) = mounts.get(&target_id) {
+        return Ok(existing.as_ref().display().to_string());
+    }
+
+    let guard = BackupMountGuard::mount(
+        TmpMountGuard::mount(
+            &target_id
+                .clone()
+                .load(ctx.secret_store.acquire().await?.as_mut())
+                .await?,
+            ReadWrite,
+        )
+        .await?,
+        &password,
+    )
+    .await?;
+
+    let res = guard.as_ref().display().to_string();
+
+    mounts.insert(target_id, guard);
+
+    Ok(res)
+}
+#[command(display(display_none))]
+#[instrument(skip_all)]
+pub async fn umount(
+    #[context] _ctx: RpcContext,
+    #[arg(rename = "target-id")] target_id: Option<BackupTargetId>,
+) -> Result<(), Error> {
+    let mut mounts = USER_MOUNTS.lock().await;
+    if let Some(target_id) = target_id {
+        if let Some(existing) = mounts.remove(&target_id) {
+            existing.unmount().await?;
+        }
+    } else {
+        for (_, existing) in std::mem::take(&mut *mounts) {
+            existing.unmount().await?;
+        }
+    }
+
+    Ok(())
+}

core/startos/src/bins/avahi_alias.rs

@@ -0,0 +1,163 @@
+use avahi_sys::{
+    self, avahi_client_errno, avahi_entry_group_add_service, avahi_entry_group_commit,
+    avahi_strerror, AvahiClient,
+};
+
+fn log_str_error(action: &str, e: i32) {
+    unsafe {
+        let e_str = avahi_strerror(e);
+        eprintln!(
+            "Could not {}: {:?}",
+            action,
+            std::ffi::CStr::from_ptr(e_str)
+        );
+    }
+}
+
+pub fn main() {
+    let aliases: Vec<_> = std::env::args().skip(1).collect();
+    unsafe {
+        let simple_poll = avahi_sys::avahi_simple_poll_new();
+        let poll = avahi_sys::avahi_simple_poll_get(simple_poll);
+        let mut box_err = Box::pin(0 as i32);
+        let err_c: *mut i32 = box_err.as_mut().get_mut();
+        let avahi_client = avahi_sys::avahi_client_new(
+            poll,
+            avahi_sys::AvahiClientFlags::AVAHI_CLIENT_NO_FAIL,
+            Some(client_callback),
+            std::ptr::null_mut(),
+            err_c,
+        );
+        if avahi_client == std::ptr::null_mut::<AvahiClient>() {
+            log_str_error("create Avahi client", *box_err);
+            panic!("Failed to create Avahi Client");
+        }
+        let group = avahi_sys::avahi_entry_group_new(
+            avahi_client,
+            Some(entry_group_callback),
+            std::ptr::null_mut(),
+        );
+        if group == std::ptr::null_mut() {
+            log_str_error("create Avahi entry group", avahi_client_errno(avahi_client));
+            panic!("Failed to create Avahi Entry Group");
+        }
+        let mut hostname_buf = vec![0];
+        let hostname_raw = avahi_sys::avahi_client_get_host_name_fqdn(avahi_client);
+        hostname_buf.extend_from_slice(std::ffi::CStr::from_ptr(hostname_raw).to_bytes_with_nul());
+        let buflen = hostname_buf.len();
+        debug_assert!(hostname_buf.ends_with(b".local\0"));
+        debug_assert!(!hostname_buf[..(buflen - 7)].contains(&b'.'));
+        // assume fixed length prefix on hostname due to local address
+        hostname_buf[0] = (buflen - 8) as u8; // set the prefix length to len - 8 (leading byte, .local, nul) for the main address
+        hostname_buf[buflen - 7] = 5; // set the prefix length to 5 for "local"
+        let mut res;
+        let http_tcp_cstr =
+            std::ffi::CString::new("_http._tcp").expect("Could not cast _http._tcp to c string");
+        res = avahi_entry_group_add_service(
+            group,
+            avahi_sys::AVAHI_IF_UNSPEC,
+            avahi_sys::AVAHI_PROTO_UNSPEC,
+            avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST,
+            hostname_raw,
+            http_tcp_cstr.as_ptr(),
+            std::ptr::null(),
+            std::ptr::null(),
+            443,
+            // below is a secret final argument that the type signature of this function does not tell you that it
+            // needs. This is because the C lib function takes a variable number of final arguments indicating the
+            // desired TXT records to add to this service entry. The way it decides when to stop taking arguments
+            // from the stack and dereferencing them is when it finds a null pointer...because fuck you, that's why.
+            // The consequence of this is that forgetting this last argument will cause segfaults or other undefined
+            // behavior. Welcome back to the stone age motherfucker.
+            std::ptr::null::<libc::c_char>(),
+        );
+        if res < avahi_sys::AVAHI_OK {
+            log_str_error("add service to Avahi entry group", res);
+            panic!("Failed to load Avahi services");
+        }
+        eprintln!("Published {:?}", std::ffi::CStr::from_ptr(hostname_raw));
+        for alias in aliases {
+            let lan_address = alias + ".local";
+            let lan_address_ptr = std::ffi::CString::new(lan_address)
+                .expect("Could not cast lan address to c string");
+            res = avahi_sys::avahi_entry_group_add_record(
+                group,
+                avahi_sys::AVAHI_IF_UNSPEC,
+                avahi_sys::AVAHI_PROTO_UNSPEC,
+                avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST
+                    | avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_ALLOW_MULTIPLE,
+                lan_address_ptr.as_ptr(),
+                avahi_sys::AVAHI_DNS_CLASS_IN as u16,
+                avahi_sys::AVAHI_DNS_TYPE_CNAME as u16,
+                avahi_sys::AVAHI_DEFAULT_TTL,
+                hostname_buf.as_ptr().cast(),
+                hostname_buf.len(),
+            );
+            if res < avahi_sys::AVAHI_OK {
+                log_str_error("add CNAME record to Avahi entry group", res);
+                panic!("Failed to load Avahi services");
+            }
+            eprintln!("Published {:?}", lan_address_ptr);
+        }
+        let commit_err = avahi_entry_group_commit(group);
+        if commit_err < avahi_sys::AVAHI_OK {
+            log_str_error("reset Avahi entry group", commit_err);
+            panic!("Failed to load Avahi services: reset");
+        }
+    }
+    std::thread::park()
+}
+
+unsafe extern "C" fn entry_group_callback(
+    _group: *mut avahi_sys::AvahiEntryGroup,
+    state: avahi_sys::AvahiEntryGroupState,
+    _userdata: *mut core::ffi::c_void,
+) {
+    match state {
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_FAILURE => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_FAILURE");
+        }
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_COLLISION => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_COLLISION");
+        }
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_UNCOMMITED => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_UNCOMMITED");
+        }
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_ESTABLISHED => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_ESTABLISHED");
+        }
+        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_REGISTERING => {
+            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_REGISTERING");
+        }
+        other => {
+            eprintln!("AvahiCallback: EntryGroupState = {}", other);
+        }
+    }
+}
+
+unsafe extern "C" fn client_callback(
+    _group: *mut avahi_sys::AvahiClient,
+    state: avahi_sys::AvahiClientState,
+    _userdata: *mut core::ffi::c_void,
+) {
+    match state {
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_FAILURE => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_FAILURE");
+        }
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_RUNNING => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_RUNNING");
+        }
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_CONNECTING => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_CONNECTING");
+        }
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_COLLISION => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_COLLISION");
+        }
+        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_REGISTERING => {
+            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_REGISTERING");
+        }
+        other => {
+            eprintln!("AvahiCallback: ClientState = {}", other);
+        }
+    }
+}

core/startos/src/bins/deprecated.rs

@@ -0,0 +1,9 @@
+pub fn renamed(old: &str, new: &str) -> ! {
+    eprintln!("{old} has been renamed to {new}");
+    std::process::exit(1)
+}
+
+pub fn removed(name: &str) -> ! {
+    eprintln!("{name} has been removed");
+    std::process::exit(1)
+}

core/startos/src/bins/mod.rs

@@ -0,0 +1,59 @@
+use std::path::Path;
+
+#[cfg(feature = "avahi-alias")]
+pub mod avahi_alias;
+pub mod deprecated;
+#[cfg(feature = "cli")]
+pub mod start_cli;
+#[cfg(feature = "js-engine")]
+pub mod start_deno;
+#[cfg(feature = "daemon")]
+pub mod start_init;
+#[cfg(feature = "sdk")]
+pub mod start_sdk;
+#[cfg(feature = "daemon")]
+pub mod startd;
+
+fn select_executable(name: &str) -> Option<fn()> {
+    match name {
+        #[cfg(feature = "avahi-alias")]
+        "avahi-alias" => Some(avahi_alias::main),
+        #[cfg(feature = "js_engine")]
+        "start-deno" => Some(start_deno::main),
+        #[cfg(feature = "cli")]
+        "start-cli" => Some(start_cli::main),
+        #[cfg(feature = "sdk")]
+        "start-sdk" => Some(start_sdk::main),
+        #[cfg(feature = "daemon")]
+        "startd" => Some(startd::main),
+        "embassy-cli" => Some(|| deprecated::renamed("embassy-cli", "start-cli")),
+        "embassy-sdk" => Some(|| deprecated::renamed("embassy-sdk", "start-sdk")),
+        "embassyd" => Some(|| deprecated::renamed("embassyd", "startd")),
+        "embassy-init" => Some(|| deprecated::removed("embassy-init")),
+        _ => None,
+    }
+}
+
+pub fn startbox() {
+    let args = std::env::args().take(2).collect::<Vec<_>>();
+    if let Some(x) = args
+        .get(0)
+        .and_then(|s| Path::new(&*s).file_name())
+        .and_then(|s| s.to_str())
+        .and_then(|s| select_executable(&s))
+    {
+        x()
+    } else if let Some(x) = args.get(1).and_then(|s| select_executable(&s)) {
+        x()
+    } else {
+        eprintln!(
+            "unknown executable: {}",
+            args.get(0)
+                .filter(|x| &**x != "startbox")
+                .or_else(|| args.get(1))
+                .map(|s| s.as_str())
+                .unwrap_or("N/A")
+        );
+        std::process::exit(1);
+    }
+}

core/startos/src/bins/start_cli.rs

@@ -0,0 +1,62 @@
+use clap::Arg;
+use rpc_toolkit::run_cli;
+use rpc_toolkit::yajrc::RpcError;
+use serde_json::Value;
+
+use crate::context::CliContext;
+use crate::util::logger::EmbassyLogger;
+use crate::version::{Current, VersionT};
+use crate::Error;
+
+lazy_static::lazy_static! {
+    static ref VERSION_STRING: String = Current::new().semver().to_string();
+}
+
+fn inner_main() -> Result<(), Error> {
+    run_cli!({
+        command: crate::main_api,
+        app: app => app
+            .name("StartOS CLI")
+            .version(&**VERSION_STRING)
+            .arg(
+                clap::Arg::with_name("config")
+                    .short('c')
+                    .long("config")
+                    .takes_value(true),
+            )
+            .arg(Arg::with_name("host").long("host").short('h').takes_value(true))
+            .arg(Arg::with_name("proxy").long("proxy").short('p').takes_value(true)),
+        context: matches => {
+            EmbassyLogger::init();
+            CliContext::init(matches)?
+        },
+        exit: |e: RpcError| {
+            match e.data {
+                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
+                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
+                    eprintln!("{}: {}", e.message, s);
+                    if let Some(Value::String(s)) = o.get("debug") {
+                        tracing::debug!("{}", s)
+                    }
+                }
+                Some(a) => eprintln!("{}: {}", e.message, a),
+                None => eprintln!("{}", e.message),
+            }
+
+            std::process::exit(e.code);
+        }
+    });
+    Ok(())
+}
+
+pub fn main() {
+    match inner_main() {
+        Ok(_) => (),
+        Err(e) => {
+            eprintln!("{}", e.source);
+            tracing::debug!("{:?}", e.source);
+            drop(e.source);
+            std::process::exit(e.kind as i32)
+        }
+    }
+}

core/startos/src/bins/start_deno.rs

@@ -0,0 +1,134 @@
+use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::{command, run_cli, Context};
+use serde_json::Value;
+
+use crate::procedure::js_scripts::ExecuteArgs;
+use crate::s9pk::manifest::PackageId;
+use crate::util::serde::{display_serializable, parse_stdin_deserializable};
+use crate::version::{Current, VersionT};
+use crate::Error;
+
+lazy_static::lazy_static! {
+    static ref VERSION_STRING: String = Current::new().semver().to_string();
+}
+
+struct DenoContext;
+impl Context for DenoContext {}
+
+#[command(subcommands(execute, sandbox))]
+fn deno_api() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(cli_only, display(display_serializable))]
+async fn execute(
+    #[arg(stdin, parse(parse_stdin_deserializable))] arg: ExecuteArgs,
+) -> Result<Result<Value, (i32, String)>, Error> {
+    let ExecuteArgs {
+        procedure,
+        directory,
+        pkg_id,
+        pkg_version,
+        name,
+        volumes,
+        input,
+    } = arg;
+    PackageLogger::init(&pkg_id);
+    procedure
+        .execute_impl(&directory, &pkg_id, &pkg_version, name, &volumes, input)
+        .await
+}
+#[command(cli_only, display(display_serializable))]
+async fn sandbox(
+    #[arg(stdin, parse(parse_stdin_deserializable))] arg: ExecuteArgs,
+) -> Result<Result<Value, (i32, String)>, Error> {
+    let ExecuteArgs {
+        procedure,
+        directory,
+        pkg_id,
+        pkg_version,
+        name,
+        volumes,
+        input,
+    } = arg;
+    PackageLogger::init(&pkg_id);
+    procedure
+        .sandboxed_impl(&directory, &pkg_id, &pkg_version, &volumes, input, name)
+        .await
+}
+
+use tracing::Subscriber;
+use tracing_subscriber::util::SubscriberInitExt;
+
+#[derive(Clone)]
+struct PackageLogger {}
+
+impl PackageLogger {
+    fn base_subscriber(id: &PackageId) -> impl Subscriber {
+        use tracing_error::ErrorLayer;
+        use tracing_subscriber::prelude::*;
+        use tracing_subscriber::{fmt, EnvFilter};
+
+        let filter_layer = EnvFilter::default().add_directive(
+            format!("{}=warn", std::module_path!().split("::").next().unwrap())
+                .parse()
+                .unwrap(),
+        );
+        let fmt_layer = fmt::layer().with_writer(std::io::stderr).with_target(true);
+        let journald_layer = tracing_journald::layer()
+            .unwrap()
+            .with_syslog_identifier(format!("{id}.embassy"));
+
+        let sub = tracing_subscriber::registry()
+            .with(filter_layer)
+            .with(fmt_layer)
+            .with(journald_layer)
+            .with(ErrorLayer::default());
+
+        sub
+    }
+    pub fn init(id: &PackageId) -> Self {
+        Self::base_subscriber(id).init();
+        color_eyre::install().unwrap_or_else(|_| tracing::warn!("tracing too many times"));
+
+        Self {}
+    }
+}
+
+fn inner_main() -> Result<(), Error> {
+    run_cli!({
+        command: deno_api,
+        app: app => app
+            .name("StartOS Deno Executor")
+            .version(&**VERSION_STRING),
+        context: _m => DenoContext,
+        exit: |e: RpcError| {
+            match e.data {
+                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
+                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
+                    eprintln!("{}: {}", e.message, s);
+                    if let Some(Value::String(s)) = o.get("debug") {
+                        tracing::debug!("{}", s)
+                    }
+                }
+                Some(a) => eprintln!("{}: {}", e.message, a),
+                None => eprintln!("{}", e.message),
+            }
+
+            std::process::exit(e.code);
+        }
+    });
+    Ok(())
+}
+
+pub fn main() {
+    match inner_main() {
+        Ok(_) => (),
+        Err(e) => {
+            eprintln!("{}", e.source);
+            tracing::debug!("{:?}", e.source);
+            drop(e.source);
+            std::process::exit(e.kind as i32)
+        }
+    }
+}

core/startos/src/bins/start_init.rs

@@ -0,0 +1,268 @@
+use std::net::{Ipv6Addr, SocketAddr};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::Duration;
+
+use tokio::process::Command;
+use tracing::instrument;
+
+use crate::context::rpc::RpcContextConfig;
+use crate::context::{DiagnosticContext, InstallContext, SetupContext};
+use crate::disk::fsck::RepairStrategy;
+use crate::disk::main::DEFAULT_PASSWORD;
+use crate::disk::REPAIR_DISK_PATH;
+use crate::firmware::update_firmware;
+use crate::init::STANDBY_MODE_PATH;
+use crate::net::web_server::WebServer;
+use crate::shutdown::Shutdown;
+use crate::sound::CHIME;
+use crate::util::Invoke;
+use crate::{Error, ErrorKind, ResultExt, PLATFORM};
+
+#[instrument(skip_all)]
+async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
+    if update_firmware().await?.0 {
+        return Ok(Some(Shutdown {
+            export_args: None,
+            restart: true,
+        }));
+    }
+
+    Command::new("ln")
+        .arg("-sf")
+        .arg("/usr/lib/startos/scripts/fake-apt")
+        .arg("/usr/local/bin/apt")
+        .invoke(crate::ErrorKind::OpenSsh)
+        .await?;
+    Command::new("ln")
+        .arg("-sf")
+        .arg("/usr/lib/startos/scripts/fake-apt")
+        .arg("/usr/local/bin/apt-get")
+        .invoke(crate::ErrorKind::OpenSsh)
+        .await?;
+    Command::new("ln")
+        .arg("-sf")
+        .arg("/usr/lib/startos/scripts/fake-apt")
+        .arg("/usr/local/bin/aptitude")
+        .invoke(crate::ErrorKind::OpenSsh)
+        .await?;
+
+    Command::new("make-ssl-cert")
+        .arg("generate-default-snakeoil")
+        .arg("--force-overwrite")
+        .invoke(crate::ErrorKind::OpenSsl)
+        .await?;
+
+    if tokio::fs::metadata("/run/live/medium").await.is_ok() {
+        Command::new("sed")
+            .arg("-i")
+            .arg("s/PasswordAuthentication no/PasswordAuthentication yes/g")
+            .arg("/etc/ssh/sshd_config")
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+        Command::new("systemctl")
+            .arg("reload")
+            .arg("ssh")
+            .invoke(crate::ErrorKind::OpenSsh)
+            .await?;
+
+        let ctx = InstallContext::init(cfg_path).await?;
+
+        let server = WebServer::install(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            ctx.clone(),
+        )
+        .await?;
+
+        tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
+        CHIME.play().await?;
+
+        ctx.shutdown
+            .subscribe()
+            .recv()
+            .await
+            .expect("context dropped");
+
+        server.shutdown().await;
+
+        Command::new("reboot")
+            .invoke(crate::ErrorKind::Unknown)
+            .await?;
+    } else if tokio::fs::metadata("/media/embassy/config/disk.guid")
+        .await
+        .is_err()
+    {
+        let ctx = SetupContext::init(cfg_path).await?;
+
+        let server = WebServer::setup(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            ctx.clone(),
+        )
+        .await?;
+
+        tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
+        CHIME.play().await?;
+        ctx.shutdown
+            .subscribe()
+            .recv()
+            .await
+            .expect("context dropped");
+
+        server.shutdown().await;
+
+        tokio::task::yield_now().await;
+        if let Err(e) = Command::new("killall")
+            .arg("firefox-esr")
+            .invoke(ErrorKind::NotFound)
+            .await
+        {
+            tracing::error!("Failed to kill kiosk: {}", e);
+            tracing::debug!("{:?}", e);
+        }
+    } else {
+        let cfg = RpcContextConfig::load(cfg_path).await?;
+        let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+            .await?;
+        let guid = guid_string.trim();
+        let requires_reboot = crate::disk::main::import(
+            guid,
+            cfg.datadir(),
+            if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
+                RepairStrategy::Aggressive
+            } else {
+                RepairStrategy::Preen
+            },
+            if guid.ends_with("_UNENC") {
+                None
+            } else {
+                Some(DEFAULT_PASSWORD)
+            },
+        )
+        .await?;
+        if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
+            tokio::fs::remove_file(REPAIR_DISK_PATH)
+                .await
+                .with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
+        }
+        if requires_reboot.0 {
+            crate::disk::main::export(guid, cfg.datadir()).await?;
+            Command::new("reboot")
+                .invoke(crate::ErrorKind::Unknown)
+                .await?;
+        }
+        tracing::info!("Loaded Disk");
+        crate::init::init(&cfg).await?;
+    }
+
+    Ok(None)
+}
+
+async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
+    let script = path.as_ref();
+    if script.exists() {
+        match Command::new("/bin/bash").arg(script).spawn() {
+            Ok(mut c) => {
+                if let Err(e) = c.wait().await {
+                    tracing::error!("Error Running {}: {}", script.display(), e);
+                    tracing::debug!("{:?}", e);
+                }
+            }
+            Err(e) => {
+                tracing::error!("Error Running {}: {}", script.display(), e);
+                tracing::debug!("{:?}", e);
+            }
+        }
+    }
+}
+
+#[instrument(skip_all)]
+async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
+    if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
+        tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
+        Command::new("sync").invoke(ErrorKind::Filesystem).await?;
+        crate::sound::SHUTDOWN.play().await?;
+        futures::future::pending::<()>().await;
+    }
+
+    crate::sound::BEP.play().await?;
+
+    run_script_if_exists("/media/embassy/config/preinit.sh").await;
+
+    let res = match setup_or_init(cfg_path.clone()).await {
+        Err(e) => {
+            async move {
+                tracing::error!("{}", e.source);
+                tracing::debug!("{}", e.source);
+                crate::sound::BEETHOVEN.play().await?;
+
+                let ctx = DiagnosticContext::init(
+                    cfg_path,
+                    if tokio::fs::metadata("/media/embassy/config/disk.guid")
+                        .await
+                        .is_ok()
+                    {
+                        Some(Arc::new(
+                            tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+                                .await?
+                                .trim()
+                                .to_owned(),
+                        ))
+                    } else {
+                        None
+                    },
+                    e,
+                )
+                .await?;
+
+                let server = WebServer::diagnostic(
+                    SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+                    ctx.clone(),
+                )
+                .await?;
+
+                let shutdown = ctx.shutdown.subscribe().recv().await.unwrap();
+
+                server.shutdown().await;
+
+                Ok(shutdown)
+            }
+            .await
+        }
+        Ok(s) => Ok(s),
+    };
+
+    run_script_if_exists("/media/embassy/config/postinit.sh").await;
+
+    res
+}
+
+pub fn main() {
+    let matches = clap::App::new("start-init")
+        .arg(
+            clap::Arg::with_name("config")
+                .short('c')
+                .long("config")
+                .takes_value(true),
+        )
+        .get_matches();
+
+    let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
+    let res = {
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .expect("failed to initialize runtime");
+        rt.block_on(inner_main(cfg_path))
+    };
+
+    match res {
+        Ok(Some(shutdown)) => shutdown.execute(),
+        Ok(None) => (),
+        Err(e) => {
+            eprintln!("{}", e.source);
+            tracing::debug!("{:?}", e.source);
+            drop(e.source);
+            std::process::exit(e.kind as i32)
+        }
+    }
+}

core/startos/src/bins/start_sdk.rs

@@ -0,0 +1,61 @@
+use rpc_toolkit::run_cli;
+use rpc_toolkit::yajrc::RpcError;
+use serde_json::Value;
+
+use crate::context::SdkContext;
+use crate::util::logger::EmbassyLogger;
+use crate::version::{Current, VersionT};
+use crate::Error;
+
+lazy_static::lazy_static! {
+    static ref VERSION_STRING: String = Current::new().semver().to_string();
+}
+
+fn inner_main() -> Result<(), Error> {
+    run_cli!({
+        command: crate::portable_api,
+        app: app => app
+            .name("StartOS SDK")
+            .version(&**VERSION_STRING)
+            .arg(
+                clap::Arg::with_name("config")
+                    .short('c')
+                    .long("config")
+                    .takes_value(true),
+            ),
+        context: matches => {
+            if let Err(_) = std::env::var("RUST_LOG") {
+                std::env::set_var("RUST_LOG", "embassy=warn,js_engine=warn");
+            }
+            EmbassyLogger::init();
+            SdkContext::init(matches)?
+        },
+        exit: |e: RpcError| {
+            match e.data {
+                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
+                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
+                    eprintln!("{}: {}", e.message, s);
+                    if let Some(Value::String(s)) = o.get("debug") {
+                        tracing::debug!("{}", s)
+                    }
+                }
+                Some(a) => eprintln!("{}: {}", e.message, a),
+                None => eprintln!("{}", e.message),
+            }
+            std::process::exit(e.code);
+        }
+    });
+    Ok(())
+}
+
+pub fn main() {
+    match inner_main() {
+        Ok(_) => (),
+        Err(e) => {
+            eprintln!("{}", e.source);
+            tracing::debug!("{:?}", e.source);
+            drop(e.source);
+            std::process::exit(e.kind as i32)
+        }
+    }
+}

core/startos/src/bins/startd.rs

@@ -0,0 +1,187 @@
+use std::net::{Ipv6Addr, SocketAddr};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use color_eyre::eyre::eyre;
+use futures::{FutureExt, TryFutureExt};
+use tokio::signal::unix::signal;
+use tracing::instrument;
+
+use crate::context::{DiagnosticContext, RpcContext};
+use crate::net::web_server::WebServer;
+use crate::shutdown::Shutdown;
+use crate::system::launch_metrics_task;
+use crate::util::logger::EmbassyLogger;
+use crate::{Error, ErrorKind, ResultExt};
+
+#[instrument(skip_all)]
+async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
+    let (rpc_ctx, server, shutdown) = async {
+        let rpc_ctx = RpcContext::init(
+            cfg_path,
+            Arc::new(
+                tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+                    .await?
+                    .trim()
+                    .to_owned(),
+            ),
+        )
+        .await?;
+        crate::hostname::sync_hostname(&rpc_ctx.account.read().await.hostname).await?;
+        let server = WebServer::main(
+            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+            rpc_ctx.clone(),
+        )
+        .await?;
+
+        let mut shutdown_recv = rpc_ctx.shutdown.subscribe();
+
+        let sig_handler_ctx = rpc_ctx.clone();
+        let sig_handler = tokio::spawn(async move {
+            use tokio::signal::unix::SignalKind;
+            futures::future::select_all(
+                [
+                    SignalKind::interrupt(),
+                    SignalKind::quit(),
+                    SignalKind::terminate(),
+                ]
+                .iter()
+                .map(|s| {
+                    async move {
+                        signal(*s)
+                            .unwrap_or_else(|_| panic!("register {:?} handler", s))
+                            .recv()
+                            .await
+                    }
+                    .boxed()
+                }),
+            )
+            .await;
+            sig_handler_ctx
+                .shutdown
+                .send(None)
+                .map_err(|_| ())
+                .expect("send shutdown signal");
+        });
+
+        let metrics_ctx = rpc_ctx.clone();
+        let metrics_task = tokio::spawn(async move {
+            launch_metrics_task(&metrics_ctx.metrics_cache, || {
+                metrics_ctx.shutdown.subscribe()
+            })
+            .await
+        });
+
+        crate::sound::CHIME.play().await?;
+
+        metrics_task
+            .map_err(|e| {
+                Error::new(
+                    eyre!("{}", e).wrap_err("Metrics daemon panicked!"),
+                    ErrorKind::Unknown,
+                )
+            })
+            .map_ok(|_| tracing::debug!("Metrics daemon Shutdown"))
+            .await?;
+
+        let shutdown = shutdown_recv
+            .recv()
+            .await
+            .with_kind(crate::ErrorKind::Unknown)?;
+
+        sig_handler.abort();
+
+        Ok::<_, Error>((rpc_ctx, server, shutdown))
+    }
+    .await?;
+    server.shutdown().await;
+    rpc_ctx.shutdown().await?;
+
+    tracing::info!("RPC Context is dropped");
+
+    Ok(shutdown)
+}
+
+pub fn main() {
+    EmbassyLogger::init();
+
+    if !Path::new("/run/embassy/initialized").exists() {
+        super::start_init::main();
+        std::fs::write("/run/embassy/initialized", "").unwrap();
+    }
+
+    let matches = clap::App::new("startd")
+        .arg(
+            clap::Arg::with_name("config")
+                .short('c')
+                .long("config")
+                .takes_value(true),
+        )
+        .get_matches();
+
+    let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
+
+    let res = {
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .expect("failed to initialize runtime");
+        rt.block_on(async {
+            match inner_main(cfg_path.clone()).await {
+                Ok(a) => Ok(a),
+                Err(e) => {
+                    async {
+                        tracing::error!("{}", e.source);
+                        tracing::debug!("{:?}", e.source);
+                        crate::sound::BEETHOVEN.play().await?;
+                        let ctx = DiagnosticContext::init(
+                            cfg_path,
+                            if tokio::fs::metadata("/media/embassy/config/disk.guid")
+                                .await
+                                .is_ok()
+                            {
+                                Some(Arc::new(
+                                    tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+                                        .await?
+                                        .trim()
+                                        .to_owned(),
+                                ))
+                            } else {
+                                None
+                            },
+                            e,
+                        )
+                        .await?;
+
+                        let server = WebServer::diagnostic(
+                            SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
+                            ctx.clone(),
+                        )
+                        .await?;
+
+                        let mut shutdown = ctx.shutdown.subscribe();
+
+                        let shutdown =
+                            shutdown.recv().await.with_kind(crate::ErrorKind::Unknown)?;
+
+                        server.shutdown().await;
+
+                        Ok::<_, Error>(shutdown)
+                    }
+                    .await
+                }
+            }
+        })
+    };
+
+    match res {
+        Ok(None) => (),
+        Ok(Some(s)) => s.execute(),
+        Err(e) => {
+            eprintln!("{}", e.source);
+            tracing::debug!("{:?}", e.source);
+            drop(e.source);
+            std::process::exit(e.kind as i32)
+        }
+    }
+}

core/startos/src/config/action.rs

@@ -0,0 +1,116 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use color_eyre::eyre::eyre;
+use models::ImageId;
+use patch_db::HasModel;
+use serde::{Deserialize, Serialize};
+use tracing::instrument;
+
+use super::{Config, ConfigSpec};
+use crate::context::RpcContext;
+use crate::dependencies::Dependencies;
+use crate::prelude::*;
+use crate::procedure::docker::DockerContainers;
+use crate::procedure::{PackageProcedure, ProcedureName};
+use crate::s9pk::manifest::PackageId;
+use crate::status::health_check::HealthCheckId;
+use crate::util::Version;
+use crate::volume::Volumes;
+use crate::{Error, ResultExt};
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct ConfigRes {
+    pub config: Option<Config>,
+    pub spec: ConfigSpec,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
+#[model = "Model<Self>"]
+pub struct ConfigActions {
+    pub get: PackageProcedure,
+    pub set: PackageProcedure,
+}
+impl ConfigActions {
+    #[instrument(skip_all)]
+    pub fn validate(
+        &self,
+        _container: &Option<DockerContainers>,
+        eos_version: &Version,
+        volumes: &Volumes,
+        image_ids: &BTreeSet<ImageId>,
+    ) -> Result<(), Error> {
+        self.get
+            .validate(eos_version, volumes, image_ids, true)
+            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Get"))?;
+        self.set
+            .validate(eos_version, volumes, image_ids, true)
+            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Set"))?;
+        Ok(())
+    }
+    #[instrument(skip_all)]
+    pub async fn get(
+        &self,
+        ctx: &RpcContext,
+        pkg_id: &PackageId,
+        pkg_version: &Version,
+        volumes: &Volumes,
+    ) -> Result<ConfigRes, Error> {
+        self.get
+            .execute(
+                ctx,
+                pkg_id,
+                pkg_version,
+                ProcedureName::GetConfig,
+                volumes,
+                None::<()>,
+                None,
+            )
+            .await
+            .and_then(|res| {
+                res.map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::ConfigGen))
+            })
+    }
+
+    #[instrument(skip_all)]
+    pub async fn set(
+        &self,
+        ctx: &RpcContext,
+        pkg_id: &PackageId,
+        pkg_version: &Version,
+        dependencies: &Dependencies,
+        volumes: &Volumes,
+        input: &Config,
+    ) -> Result<SetResult, Error> {
+        let res: SetResult = self
+            .set
+            .execute(
+                ctx,
+                pkg_id,
+                pkg_version,
+                ProcedureName::SetConfig,
+                volumes,
+                Some(input),
+                None,
+            )
+            .await
+            .and_then(|res| {
+                res.map_err(|e| {
+                    Error::new(eyre!("{}", e.1), crate::ErrorKind::ConfigRulesViolation)
+                })
+            })?;
+        Ok(SetResult {
+            depends_on: res
+                .depends_on
+                .into_iter()
+                .filter(|(pkg, _)| dependencies.0.contains_key(pkg))
+                .collect(),
+        })
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SetResult {
+    pub depends_on: BTreeMap<PackageId, BTreeSet<HealthCheckId>>,
+}

core/startos/src/config/mod.rs

@@ -0,0 +1,287 @@
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+
+use color_eyre::eyre::eyre;
+use indexmap::IndexSet;
+use itertools::Itertools;
+use models::{ErrorKind, OptionExt};
+use patch_db::value::InternedString;
+use patch_db::Value;
+use regex::Regex;
+use rpc_toolkit::command;
+use tracing::instrument;
+
+use crate::context::RpcContext;
+use crate::prelude::*;
+use crate::s9pk::manifest::PackageId;
+use crate::util::display_none;
+use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
+use crate::Error;
+
+pub mod action;
+pub mod spec;
+pub mod util;
+
+pub use spec::{ConfigSpec, Defaultable};
+use util::NumRange;
+
+use self::action::ConfigRes;
+use self::spec::ValueSpecPointer;
+
+pub type Config = patch_db::value::InOMap<InternedString, Value>;
+pub trait TypeOf {
+    fn type_of(&self) -> &'static str;
+}
+impl TypeOf for Value {
+    fn type_of(&self) -> &'static str {
+        match self {
+            Value::Array(_) => "list",
+            Value::Bool(_) => "boolean",
+            Value::Null => "null",
+            Value::Number(_) => "number",
+            Value::Object(_) => "object",
+            Value::String(_) => "string",
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum ConfigurationError {
+    #[error("Timeout Error")]
+    TimeoutError(#[from] TimeoutError),
+    #[error("No Match: {0}")]
+    NoMatch(#[from] NoMatchWithPath),
+    #[error("System Error: {0}")]
+    SystemError(Error),
+    #[error("Permission Denied: {0}")]
+    PermissionDenied(ValueSpecPointer),
+}
+impl From<ConfigurationError> for Error {
+    fn from(err: ConfigurationError) -> Self {
+        let kind = match &err {
+            ConfigurationError::SystemError(e) => e.kind,
+            _ => crate::ErrorKind::ConfigGen,
+        };
+        crate::Error::new(err, kind)
+    }
+}
+
+#[derive(Clone, Copy, Debug, thiserror::Error)]
+#[error("Timeout Error")]
+pub struct TimeoutError;
+
+#[derive(Clone, Debug, thiserror::Error)]
+pub struct NoMatchWithPath {
+    pub path: Vec<InternedString>,
+    pub error: MatchError,
+}
+impl NoMatchWithPath {
+    pub fn new(error: MatchError) -> Self {
+        NoMatchWithPath {
+            path: Vec::new(),
+            error,
+        }
+    }
+    pub fn prepend(mut self, seg: InternedString) -> Self {
+        self.path.push(seg);
+        self
+    }
+}
+impl std::fmt::Display for NoMatchWithPath {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}: {}", self.path.iter().rev().join("."), self.error)
+    }
+}
+impl From<NoMatchWithPath> for Error {
+    fn from(e: NoMatchWithPath) -> Self {
+        ConfigurationError::from(e).into()
+    }
+}
+
+#[derive(Clone, Debug, thiserror::Error)]
+pub enum MatchError {
+    #[error("String {0:?} Does Not Match Pattern {1}")]
+    Pattern(Arc<String>, Regex),
+    #[error("String {0:?} Is Not In Enum {1:?}")]
+    Enum(Arc<String>, IndexSet<String>),
+    #[error("Field Is Not Nullable")]
+    NotNullable,
+    #[error("Length Mismatch: expected {0}, actual: {1}")]
+    LengthMismatch(NumRange<usize>, usize),
+    #[error("Invalid Type: expected {0}, actual: {1}")]
+    InvalidType(&'static str, &'static str),
+    #[error("Number Out Of Range: expected {0}, actual: {1}")]
+    OutOfRange(NumRange<f64>, f64),
+    #[error("Number Is Not Integral: {0}")]
+    NonIntegral(f64),
+    #[error("Variant {0:?} Is Not In Union {1:?}")]
+    Union(Arc<String>, IndexSet<String>),
+    #[error("Variant Is Missing Tag {0:?}")]
+    MissingTag(InternedString),
+    #[error("Property {0:?} Of Variant {1:?} Conflicts With Union Tag")]
+    PropertyMatchesUnionTag(InternedString, String),
+    #[error("Name of Property {0:?} Conflicts With Map Tag Name")]
+    PropertyNameMatchesMapTag(String),
+    #[error("Pointer Is Invalid: {0}")]
+    InvalidPointer(spec::ValueSpecPointer),
+    #[error("Object Key Is Invalid: {0}")]
+    InvalidKey(String),
+    #[error("Value In List Is Not Unique")]
+    ListUniquenessViolation,
+}
+
+#[command(rename = "config-spec", cli_only, blocking, display(display_none))]
+pub fn verify_spec(#[arg] path: PathBuf) -> Result<(), Error> {
+    let mut file = std::fs::File::open(&path)?;
+    let format = match path.extension().and_then(|s| s.to_str()) {
+        Some("yaml") | Some("yml") => IoFormat::Yaml,
+        Some("json") => IoFormat::Json,
+        Some("toml") => IoFormat::Toml,
+        Some("cbor") => IoFormat::Cbor,
+        _ => {
+            return Err(Error::new(
+                eyre!("Unknown file format. Expected one of yaml, json, toml, cbor."),
+                crate::ErrorKind::Deserialization,
+            ));
+        }
+    };
+    let _: ConfigSpec = format.from_reader(&mut file)?;
+
+    Ok(())
+}
+
+#[command(subcommands(get, set))]
+pub fn config(#[arg] id: PackageId) -> Result<PackageId, Error> {
+    Ok(id)
+}
+
+#[command(display(display_serializable))]
+#[instrument(skip_all)]
+pub async fn get(
+    #[context] ctx: RpcContext,
+    #[parent_data] id: PackageId,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+) -> Result<ConfigRes, Error> {
+    let db = ctx.db.peek().await;
+    let manifest = db
+        .as_package_data()
+        .as_idx(&id)
+        .or_not_found(&id)?
+        .as_installed()
+        .or_not_found(&id)?
+        .as_manifest();
+    let action = manifest
+        .as_config()
+        .de()?
+        .ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
+
+    let volumes = manifest.as_volumes().de()?;
+    let version = manifest.as_version().de()?;
+    action.get(&ctx, &id, &version, &volumes).await
+}
+
+#[command(
+    subcommands(self(set_impl(async, context(RpcContext))), set_dry),
+    display(display_none),
+    metadata(sync_db = true)
+)]
+#[instrument(skip_all)]
+pub fn set(
+    #[parent_data] id: PackageId,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+    #[arg(long = "timeout")] timeout: Option<crate::util::serde::Duration>,
+    #[arg(stdin, parse(parse_stdin_deserializable))] config: Option<Config>,
+) -> Result<(PackageId, Option<Config>, Option<Duration>), Error> {
+    Ok((id, config, timeout.map(|d| *d)))
+}
+
+#[command(rename = "dry", display(display_serializable))]
+#[instrument(skip_all)]
+pub async fn set_dry(
+    #[context] ctx: RpcContext,
+    #[parent_data] (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
+) -> Result<BTreeMap<PackageId, String>, Error> {
+    let breakages = BTreeMap::new();
+    let overrides = Default::default();
+
+    let configure_context = ConfigureContext {
+        breakages,
+        timeout,
+        config,
+        dry_run: true,
+        overrides,
+    };
+    let breakages = configure(&ctx, &id, configure_context).await?;
+
+    Ok(breakages)
+}
+
+pub struct ConfigureContext {
+    pub breakages: BTreeMap<PackageId, String>,
+    pub timeout: Option<Duration>,
+    pub config: Option<Config>,
+    pub overrides: BTreeMap<PackageId, Config>,
+    pub dry_run: bool,
+}
+
+#[instrument(skip_all)]
+pub async fn set_impl(
+    ctx: RpcContext,
+    (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
+) -> Result<(), Error> {
+    let breakages = BTreeMap::new();
+    let overrides = Default::default();
+
+    let configure_context = ConfigureContext {
+        breakages,
+        timeout,
+        config,
+        dry_run: false,
+        overrides,
+    };
+    configure(&ctx, &id, configure_context).await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn configure(
+    ctx: &RpcContext,
+    id: &PackageId,
+    configure_context: ConfigureContext,
+) -> Result<BTreeMap<PackageId, String>, Error> {
+    let db = ctx.db.peek().await;
+    let package = db
+        .as_package_data()
+        .as_idx(id)
+        .or_not_found(&id)?
+        .as_installed()
+        .or_not_found(&id)?;
+    let version = package.as_manifest().as_version().de()?;
+    ctx.managers
+        .get(&(id.clone(), version.clone()))
+        .await
+        .ok_or_else(|| {
+            Error::new(
+                eyre!("There is no manager running for {id:?} and {version:?}"),
+                ErrorKind::Unknown,
+            )
+        })?
+        .configure(configure_context)
+        .await
+}
+
+macro_rules! not_found {
+    ($x:expr) => {
+        crate::Error::new(
+            color_eyre::eyre::eyre!("Could not find {} at {}:{}", $x, module_path!(), line!()),
+            crate::ErrorKind::Incoherent,
+        )
+    };
+}
+pub(crate) use not_found;

core/startos/src/config/util.rs

@@ -0,0 +1,406 @@
+use std::borrow::Cow;
+use std::ops::{Bound, RangeBounds, RangeInclusive};
+
+use patch_db::Value;
+use rand::distributions::Distribution;
+use rand::Rng;
+
+use super::Config;
+
+pub const STATIC_NULL: Value = Value::Null;
+
+#[derive(Clone, Debug)]
+pub struct CharSet(pub Vec<(RangeInclusive<char>, usize)>, usize);
+impl CharSet {
+    pub fn contains(&self, c: &char) -> bool {
+        self.0.iter().any(|r| r.0.contains(c))
+    }
+    pub fn gen<R: Rng>(&self, rng: &mut R) -> char {
+        let mut idx = rng.gen_range(0..self.1);
+        for r in &self.0 {
+            if idx < r.1 {
+                return std::convert::TryFrom::try_from(
+                    rand::distributions::Uniform::new_inclusive(
+                        u32::from(*r.0.start()),
+                        u32::from(*r.0.end()),
+                    )
+                    .sample(rng),
+                )
+                .unwrap();
+            } else {
+                idx -= r.1;
+            }
+        }
+        unreachable!()
+    }
+}
+impl Default for CharSet {
+    fn default() -> Self {
+        CharSet(vec![('!'..='~', 94)], 94)
+    }
+}
+impl<'de> serde::de::Deserialize<'de> for CharSet {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let mut res = Vec::new();
+        let mut len = 0;
+        let mut a: Option<char> = None;
+        let mut b: Option<char> = None;
+        let mut in_range = false;
+        for c in s.chars() {
+            match c {
+                ',' => match (a, b, in_range) {
+                    (Some(start), Some(end), _) => {
+                        if !end.is_ascii() {
+                            return Err(serde::de::Error::custom("Invalid Character"));
+                        }
+                        if start >= end {
+                            return Err(serde::de::Error::custom("Invalid Bounds"));
+                        }
+                        let l = u32::from(end) - u32::from(start) + 1;
+                        res.push((start..=end, l as usize));
+                        len += l as usize;
+                        a = None;
+                        b = None;
+                        in_range = false;
+                    }
+                    (Some(start), None, false) => {
+                        len += 1;
+                        res.push((start..=start, 1));
+                        a = None;
+                    }
+                    (Some(_), None, true) => {
+                        b = Some(',');
+                    }
+                    (None, None, false) => {
+                        a = Some(',');
+                    }
+                    _ => {
+                        return Err(serde::de::Error::custom("Syntax Error"));
+                    }
+                },
+                '-' => {
+                    if a.is_none() {
+                        a = Some('-');
+                    } else if !in_range {
+                        in_range = true;
+                    } else if b.is_none() {
+                        b = Some('-')
+                    } else {
+                        return Err(serde::de::Error::custom("Syntax Error"));
+                    }
+                }
+                _ => {
+                    if a.is_none() {
+                        a = Some(c);
+                    } else if in_range && b.is_none() {
+                        b = Some(c);
+                    } else {
+                        return Err(serde::de::Error::custom("Syntax Error"));
+                    }
+                }
+            }
+        }
+        match (a, b) {
+            (Some(start), Some(end)) => {
+                if !end.is_ascii() {
+                    return Err(serde::de::Error::custom("Invalid Character"));
+                }
+                if start >= end {
+                    return Err(serde::de::Error::custom("Invalid Bounds"));
+                }
+                let l = u32::from(end) - u32::from(start) + 1;
+                res.push((start..=end, l as usize));
+                len += l as usize;
+            }
+            (Some(c), None) => {
+                len += 1;
+                res.push((c..=c, 1));
+            }
+            _ => (),
+        }
+
+        Ok(CharSet(res, len))
+    }
+}
+impl serde::ser::Serialize for CharSet {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::ser::Serializer,
+    {
+        <&str>::serialize(
+            &self
+                .0
+                .iter()
+                .map(|r| match r.1 {
+                    1 => format!("{}", r.0.start()),
+                    _ => format!("{}-{}", r.0.start(), r.0.end()),
+                })
+                .collect::<Vec<_>>()
+                .join(",")
+                .as_str(),
+            serializer,
+        )
+    }
+}
+
+pub trait MergeWith {
+    fn merge_with(&mut self, other: &serde_json::Value);
+}
+
+impl MergeWith for serde_json::Value {
+    fn merge_with(&mut self, other: &serde_json::Value) {
+        use serde_json::Value::Object;
+        if let (Object(orig), Object(ref other)) = (self, other) {
+            for (key, val) in other.into_iter() {
+                match (orig.get_mut(key), val) {
+                    (Some(new_orig @ Object(_)), other @ Object(_)) => {
+                        new_orig.merge_with(other);
+                    }
+                    (None, _) => {
+                        orig.insert(key.clone(), val.clone());
+                    }
+                    _ => (),
+                }
+            }
+        }
+    }
+}
+
+#[test]
+fn merge_with_tests() {
+    use serde_json::json;
+
+    let mut a = json!(
+        {"a": 1, "c": {"d": "123"}, "i": [1,2,3], "j": {}, "k":[1,2,3], "l": "test"}
+    );
+    a.merge_with(
+        &json!({"a":"a", "b": "b", "c":{"d":"d", "e":"e"}, "f":{"g":"g"}, "h": [1,2,3], "i":"i", "j":[1,2,3], "k":{}}),
+    );
+    assert_eq!(
+        a,
+        json!({"a": 1, "c": {"d": "123", "e":"e"}, "b":"b", "f": {"g":"g"}, "h":[1,2,3], "i":[1,2,3], "j": {}, "k":[1,2,3], "l": "test"})
+    )
+}
+pub mod serde_regex {
+    use regex::Regex;
+    use serde::*;
+
+    pub fn serialize<S>(regex: &Regex, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        <&str>::serialize(&regex.as_str(), serializer)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Regex, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        Regex::new(&s).map_err(|e| de::Error::custom(e))
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct NumRange<T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd>(
+    pub (Bound<T>, Bound<T>),
+);
+impl<T> std::ops::Deref for NumRange<T>
+where
+    T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd,
+{
+    type Target = (Bound<T>, Bound<T>);
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl<'de, T> serde::de::Deserialize<'de> for NumRange<T>
+where
+    T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd,
+    <T as std::str::FromStr>::Err: std::fmt::Display,
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let mut split = s.split(",");
+        let start = split
+            .next()
+            .map(|s| match s.get(..1) {
+                Some("(") => match s.get(1..2) {
+                    Some("*") => Ok(Bound::Unbounded),
+                    _ => s[1..]
+                        .trim()
+                        .parse()
+                        .map(Bound::Excluded)
+                        .map_err(|e| serde::de::Error::custom(e)),
+                },
+                Some("[") => s[1..]
+                    .trim()
+                    .parse()
+                    .map(Bound::Included)
+                    .map_err(|e| serde::de::Error::custom(e)),
+                _ => Err(serde::de::Error::custom(format!(
+                    "Could not parse left bound: {}",
+                    s
+                ))),
+            })
+            .transpose()?
+            .unwrap();
+        let end = split
+            .next()
+            .map(|s| match s.get(s.len() - 1..) {
+                Some(")") => match s.get(s.len() - 2..s.len() - 1) {
+                    Some("*") => Ok(Bound::Unbounded),
+                    _ => s[..s.len() - 1]
+                        .trim()
+                        .parse()
+                        .map(Bound::Excluded)
+                        .map_err(|e| serde::de::Error::custom(e)),
+                },
+                Some("]") => s[..s.len() - 1]
+                    .trim()
+                    .parse()
+                    .map(Bound::Included)
+                    .map_err(|e| serde::de::Error::custom(e)),
+                _ => Err(serde::de::Error::custom(format!(
+                    "Could not parse right bound: {}",
+                    s
+                ))),
+            })
+            .transpose()?
+            .unwrap_or(Bound::Unbounded);
+
+        Ok(NumRange((start, end)))
+    }
+}
+impl<T> std::fmt::Display for NumRange<T>
+where
+    T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.start_bound() {
+            Bound::Excluded(n) => write!(f, "({},", n)?,
+            Bound::Included(n) => write!(f, "[{},", n)?,
+            Bound::Unbounded => write!(f, "(*,")?,
+        };
+        match self.end_bound() {
+            Bound::Excluded(n) => write!(f, "{})", n),
+            Bound::Included(n) => write!(f, "{}]", n),
+            Bound::Unbounded => write!(f, "*)"),
+        }
+    }
+}
+impl<T> serde::ser::Serialize for NumRange<T>
+where
+    T: std::str::FromStr + std::fmt::Display + std::cmp::PartialOrd,
+{
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::ser::Serializer,
+    {
+        <&str>::serialize(&format!("{}", self).as_str(), serializer)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub enum UniqueBy {
+    Any(Vec<UniqueBy>),
+    All(Vec<UniqueBy>),
+    Exactly(String),
+    NotUnique,
+}
+impl UniqueBy {
+    pub fn eq(&self, lhs: &Config, rhs: &Config) -> bool {
+        match self {
+            UniqueBy::Any(any) => any.iter().any(|u| u.eq(lhs, rhs)),
+            UniqueBy::All(all) => all.iter().all(|u| u.eq(lhs, rhs)),
+            UniqueBy::Exactly(key) => lhs.get(&**key) == rhs.get(&**key),
+            UniqueBy::NotUnique => false,
+        }
+    }
+}
+impl Default for UniqueBy {
+    fn default() -> Self {
+        UniqueBy::NotUnique
+    }
+}
+impl<'de> serde::de::Deserialize<'de> for UniqueBy {
+    fn deserialize<D: serde::de::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        struct Visitor;
+        impl<'de> serde::de::Visitor<'de> for Visitor {
+            type Value = UniqueBy;
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                write!(formatter, "a key, an \"any\" object, or an \"all\" object")
+            }
+            fn visit_str<E: serde::de::Error>(self, v: &str) -> Result<Self::Value, E> {
+                Ok(UniqueBy::Exactly(v.to_owned()))
+            }
+            fn visit_string<E: serde::de::Error>(self, v: String) -> Result<Self::Value, E> {
+                Ok(UniqueBy::Exactly(v))
+            }
+            fn visit_map<A: serde::de::MapAccess<'de>>(
+                self,
+                mut map: A,
+            ) -> Result<Self::Value, A::Error> {
+                let mut variant = None;
+                while let Some(key) = map.next_key::<Cow<str>>()? {
+                    match key.as_ref() {
+                        "any" => {
+                            return Ok(UniqueBy::Any(map.next_value()?));
+                        }
+                        "all" => {
+                            return Ok(UniqueBy::All(map.next_value()?));
+                        }
+                        _ => {
+                            variant = Some(key);
+                        }
+                    }
+                }
+                Err(serde::de::Error::unknown_variant(
+                    variant.unwrap_or_default().as_ref(),
+                    &["any", "all"],
+                ))
+            }
+            fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> {
+                Ok(UniqueBy::NotUnique)
+            }
+            fn visit_none<E: serde::de::Error>(self) -> Result<Self::Value, E> {
+                Ok(UniqueBy::NotUnique)
+            }
+        }
+        deserializer.deserialize_any(Visitor)
+    }
+}
+
+impl serde::ser::Serialize for UniqueBy {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::ser::Serializer,
+    {
+        use serde::ser::SerializeMap;
+
+        match self {
+            UniqueBy::Any(any) => {
+                let mut map = serializer.serialize_map(Some(1))?;
+                map.serialize_key("any")?;
+                map.serialize_value(any)?;
+                map.end()
+            }
+            UniqueBy::All(all) => {
+                let mut map = serializer.serialize_map(Some(1))?;
+                map.serialize_key("all")?;
+                map.serialize_value(all)?;
+                map.end()
+            }
+            UniqueBy::Exactly(key) => serializer.serialize_str(key),
+            UniqueBy::NotUnique => serializer.serialize_unit(),
+        }
+    }
+}

core/startos/src/context/cli.rs

@@ -0,0 +1,185 @@
+use std::fs::File;
+use std::io::BufReader;
+use std::net::Ipv4Addr;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use cookie_store::{CookieStore, RawCookie};
+use josekit::jwk::Jwk;
+use reqwest::Proxy;
+use reqwest_cookie_store::CookieStoreMutex;
+use rpc_toolkit::reqwest::{Client, Url};
+use rpc_toolkit::url::Host;
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tracing::instrument;
+
+use super::setup::CURRENT_SECRET;
+use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
+use crate::util::config::{load_config_from_paths, local_config_path};
+use crate::ResultExt;
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct CliContextConfig {
+    pub host: Option<Url>,
+    #[serde(deserialize_with = "crate::util::serde::deserialize_from_str_opt")]
+    #[serde(default)]
+    pub proxy: Option<Url>,
+    pub cookie_path: Option<PathBuf>,
+}
+
+#[derive(Debug)]
+pub struct CliContextSeed {
+    pub base_url: Url,
+    pub rpc_url: Url,
+    pub client: Client,
+    pub cookie_store: Arc<CookieStoreMutex>,
+    pub cookie_path: PathBuf,
+}
+impl Drop for CliContextSeed {
+    fn drop(&mut self) {
+        let tmp = format!("{}.tmp", self.cookie_path.display());
+        let parent_dir = self.cookie_path.parent().unwrap_or(Path::new("/"));
+        if !parent_dir.exists() {
+            std::fs::create_dir_all(&parent_dir).unwrap();
+        }
+        let mut writer = fd_lock_rs::FdLock::lock(
+            File::create(&tmp).unwrap(),
+            fd_lock_rs::LockType::Exclusive,
+            true,
+        )
+        .unwrap();
+        let mut store = self.cookie_store.lock().unwrap();
+        store.remove("localhost", "", "local");
+        store.save_json(&mut *writer).unwrap();
+        writer.sync_all().unwrap();
+        std::fs::rename(tmp, &self.cookie_path).unwrap();
+    }
+}
+
+const DEFAULT_HOST: Host<&'static str> = Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1));
+const DEFAULT_PORT: u16 = 5959;
+
+#[derive(Debug, Clone)]
+pub struct CliContext(Arc<CliContextSeed>);
+impl CliContext {
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
+        let local_config_path = local_config_path();
+        let base: CliContextConfig = load_config_from_paths(
+            matches
+                .values_of("config")
+                .into_iter()
+                .flatten()
+                .map(|p| Path::new(p))
+                .chain(local_config_path.as_deref().into_iter())
+                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+        )?;
+        let mut url = if let Some(host) = matches.value_of("host") {
+            host.parse()?
+        } else if let Some(host) = base.host {
+            host
+        } else {
+            "http://localhost".parse()?
+        };
+        let proxy = if let Some(proxy) = matches.value_of("proxy") {
+            Some(proxy.parse()?)
+        } else {
+            base.proxy
+        };
+
+        let cookie_path = base.cookie_path.unwrap_or_else(|| {
+            local_config_path
+                .as_deref()
+                .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
+                .parent()
+                .unwrap_or(Path::new("/"))
+                .join(".cookies.json")
+        });
+        let cookie_store = Arc::new(CookieStoreMutex::new({
+            let mut store = if cookie_path.exists() {
+                CookieStore::load_json(BufReader::new(File::open(&cookie_path)?))
+                    .map_err(|e| eyre!("{}", e))
+                    .with_kind(crate::ErrorKind::Deserialization)?
+            } else {
+                CookieStore::default()
+            };
+            if let Ok(local) = std::fs::read_to_string(LOCAL_AUTH_COOKIE_PATH) {
+                store
+                    .insert_raw(
+                        &RawCookie::new("local", local),
+                        &"http://localhost".parse()?,
+                    )
+                    .with_kind(crate::ErrorKind::Network)?;
+            }
+            store
+        }));
+
+        Ok(CliContext(Arc::new(CliContextSeed {
+            base_url: url.clone(),
+            rpc_url: {
+                url.path_segments_mut()
+                    .map_err(|_| eyre!("Url cannot be base"))
+                    .with_kind(crate::ErrorKind::ParseUrl)?
+                    .push("rpc")
+                    .push("v1");
+                url
+            },
+            client: {
+                let mut builder = Client::builder().cookie_provider(cookie_store.clone());
+                if let Some(proxy) = proxy {
+                    builder =
+                        builder.proxy(Proxy::all(proxy).with_kind(crate::ErrorKind::ParseUrl)?)
+                }
+                builder.build().expect("cannot fail")
+            },
+            cookie_store,
+            cookie_path,
+        })))
+    }
+}
+impl AsRef<Jwk> for CliContext {
+    fn as_ref(&self) -> &Jwk {
+        &*CURRENT_SECRET
+    }
+}
+impl std::ops::Deref for CliContext {
+    type Target = CliContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl Context for CliContext {
+    fn protocol(&self) -> &str {
+        self.0.base_url.scheme()
+    }
+    fn host(&self) -> Host<&str> {
+        self.0.base_url.host().unwrap_or(DEFAULT_HOST)
+    }
+    fn port(&self) -> u16 {
+        self.0.base_url.port().unwrap_or(DEFAULT_PORT)
+    }
+    fn path(&self) -> &str {
+        self.0.rpc_url.path()
+    }
+    fn url(&self) -> Url {
+        self.0.rpc_url.clone()
+    }
+    fn client(&self) -> &Client {
+        &self.0.client
+    }
+}
+/// When we had an empty proxy the system wasn't working like it used to, which allowed empty proxy
+#[test]
+fn test_cli_proxy_empty() {
+    serde_yaml::from_str::<CliContextConfig>(
+        "
+        bind_rpc:
+    ",
+    )
+    .unwrap();
+}

core/startos/src/context/diagnostic.rs

@@ -0,0 +1,83 @@
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tokio::sync::broadcast::Sender;
+use tracing::instrument;
+
+use crate::shutdown::Shutdown;
+use crate::util::config::load_config_from_paths;
+use crate::Error;
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DiagnosticContextConfig {
+    pub datadir: Option<PathBuf>,
+}
+impl DiagnosticContextConfig {
+    #[instrument(skip_all)]
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+
+    pub fn datadir(&self) -> &Path {
+        self.datadir
+            .as_deref()
+            .unwrap_or_else(|| Path::new("/embassy-data"))
+    }
+}
+
+pub struct DiagnosticContextSeed {
+    pub datadir: PathBuf,
+    pub shutdown: Sender<Option<Shutdown>>,
+    pub error: Arc<RpcError>,
+    pub disk_guid: Option<Arc<String>>,
+}
+
+#[derive(Clone)]
+pub struct DiagnosticContext(Arc<DiagnosticContextSeed>);
+impl DiagnosticContext {
+    #[instrument(skip_all)]
+    pub async fn init<P: AsRef<Path> + Send + 'static>(
+        path: Option<P>,
+        disk_guid: Option<Arc<String>>,
+        error: Error,
+    ) -> Result<Self, Error> {
+        tracing::error!("Error: {}: Starting diagnostic UI", error);
+        tracing::debug!("{:?}", error);
+
+        let cfg = DiagnosticContextConfig::load(path).await?;
+
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+
+        Ok(Self(Arc::new(DiagnosticContextSeed {
+            datadir: cfg.datadir().to_owned(),
+            shutdown,
+            disk_guid,
+            error: Arc::new(error.into()),
+        })))
+    }
+}
+
+impl Context for DiagnosticContext {}
+impl Deref for DiagnosticContext {
+    type Target = DiagnosticContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}

core/startos/src/context/install.rs

@@ -0,0 +1,58 @@
+use std::ops::Deref;
+use std::path::Path;
+use std::sync::Arc;
+
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tokio::sync::broadcast::Sender;
+use tracing::instrument;
+
+use crate::net::utils::find_eth_iface;
+use crate::util::config::load_config_from_paths;
+use crate::Error;
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct InstallContextConfig {}
+impl InstallContextConfig {
+    #[instrument(skip_all)]
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+}
+
+pub struct InstallContextSeed {
+    pub ethernet_interface: String,
+    pub shutdown: Sender<()>,
+}
+
+#[derive(Clone)]
+pub struct InstallContext(Arc<InstallContextSeed>);
+impl InstallContext {
+    #[instrument(skip_all)]
+    pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        let _cfg = InstallContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        Ok(Self(Arc::new(InstallContextSeed {
+            ethernet_interface: find_eth_iface().await?,
+            shutdown,
+        })))
+    }
+}
+
+impl Context for InstallContext {}
+impl Deref for InstallContext {
+    type Target = InstallContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}

core/startos/src/context/mod.rs

@@ -0,0 +1,44 @@
+pub mod cli;
+pub mod diagnostic;
+pub mod install;
+pub mod rpc;
+pub mod sdk;
+pub mod setup;
+
+pub use cli::CliContext;
+pub use diagnostic::DiagnosticContext;
+pub use install::InstallContext;
+pub use rpc::RpcContext;
+pub use sdk::SdkContext;
+pub use setup::SetupContext;
+
+impl From<CliContext> for () {
+    fn from(_: CliContext) -> Self {
+        ()
+    }
+}
+impl From<DiagnosticContext> for () {
+    fn from(_: DiagnosticContext) -> Self {
+        ()
+    }
+}
+impl From<RpcContext> for () {
+    fn from(_: RpcContext) -> Self {
+        ()
+    }
+}
+impl From<SdkContext> for () {
+    fn from(_: SdkContext) -> Self {
+        ()
+    }
+}
+impl From<SetupContext> for () {
+    fn from(_: SetupContext) -> Self {
+        ()
+    }
+}
+impl From<InstallContext> for () {
+    fn from(_: InstallContext) -> Self {
+        ()
+    }
+}

core/startos/src/context/rpc.rs

@@ -0,0 +1,466 @@
+use std::collections::BTreeMap;
+use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use helpers::to_tmp_path;
+use josekit::jwk::Jwk;
+use patch_db::json_ptr::JsonPointer;
+use patch_db::PatchDb;
+use reqwest::{Client, Proxy, Url};
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use sqlx::postgres::PgConnectOptions;
+use sqlx::PgPool;
+use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
+use tokio::time::Instant;
+use tracing::instrument;
+
+use super::setup::CURRENT_SECRET;
+use crate::account::AccountInfo;
+use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation};
+use crate::db::model::{CurrentDependents, Database, PackageDataEntryMatchModelRef};
+use crate::db::prelude::PatchDbExt;
+use crate::dependencies::compute_dependency_config_errs;
+use crate::disk::OsPartitionInfo;
+use crate::init::init_postgres;
+use crate::install::cleanup::{cleanup_failed, uninstall};
+use crate::manager::ManagerMap;
+use crate::middleware::auth::HashSessionToken;
+use crate::net::net_controller::NetController;
+use crate::net::ssl::{root_ca_start_time, SslManager};
+use crate::net::wifi::WpaCli;
+use crate::notifications::NotificationManager;
+use crate::shutdown::Shutdown;
+use crate::status::MainStatus;
+use crate::system::get_mem_info;
+use crate::util::config::load_config_from_paths;
+use crate::util::lshw::{lshw, LshwDevice};
+use crate::{Error, ErrorKind, ResultExt};
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct RpcContextConfig {
+    pub wifi_interface: Option<String>,
+    pub ethernet_interface: String,
+    pub os_partitions: OsPartitionInfo,
+    pub migration_batch_rows: Option<usize>,
+    pub migration_prefetch_rows: Option<usize>,
+    pub bind_rpc: Option<SocketAddr>,
+    pub tor_control: Option<SocketAddr>,
+    pub tor_socks: Option<SocketAddr>,
+    pub dns_bind: Option<Vec<SocketAddr>>,
+    pub revision_cache_size: Option<usize>,
+    pub datadir: Option<PathBuf>,
+    pub log_server: Option<Url>,
+}
+impl RpcContextConfig {
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+    pub fn datadir(&self) -> &Path {
+        self.datadir
+            .as_deref()
+            .unwrap_or_else(|| Path::new("/embassy-data"))
+    }
+    pub async fn db(&self, account: &AccountInfo) -> Result<PatchDb, Error> {
+        let db_path = self.datadir().join("main").join("embassy.db");
+        let db = PatchDb::open(&db_path)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
+        if !db.exists(&<JsonPointer>::default()).await {
+            db.put(&<JsonPointer>::default(), &Database::init(account))
+                .await?;
+        }
+        Ok(db)
+    }
+    #[instrument(skip_all)]
+    pub async fn secret_store(&self) -> Result<PgPool, Error> {
+        init_postgres(self.datadir()).await?;
+        let secret_store =
+            PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
+                .await?;
+        sqlx::migrate!()
+            .run(&secret_store)
+            .await
+            .with_kind(crate::ErrorKind::Database)?;
+        Ok(secret_store)
+    }
+}
+
+pub struct RpcContextSeed {
+    is_closed: AtomicBool,
+    pub os_partitions: OsPartitionInfo,
+    pub wifi_interface: Option<String>,
+    pub ethernet_interface: String,
+    pub datadir: PathBuf,
+    pub disk_guid: Arc<String>,
+    pub db: PatchDb,
+    pub secret_store: PgPool,
+    pub account: RwLock<AccountInfo>,
+    pub net_controller: Arc<NetController>,
+    pub managers: ManagerMap,
+    pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
+    pub shutdown: broadcast::Sender<Option<Shutdown>>,
+    pub tor_socks: SocketAddr,
+    pub notification_manager: NotificationManager,
+    pub open_authed_websockets: Mutex<BTreeMap<HashSessionToken, Vec<oneshot::Sender<()>>>>,
+    pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
+    pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
+    pub current_secret: Arc<Jwk>,
+    pub client: Client,
+    pub hardware: Hardware,
+    pub start_time: Instant,
+}
+
+pub struct Hardware {
+    pub devices: Vec<LshwDevice>,
+    pub ram: u64,
+}
+
+#[derive(Clone)]
+pub struct RpcContext(Arc<RpcContextSeed>);
+impl RpcContext {
+    #[instrument(skip_all)]
+    pub async fn init<P: AsRef<Path> + Send + Sync + 'static>(
+        cfg_path: Option<P>,
+        disk_guid: Arc<String>,
+    ) -> Result<Self, Error> {
+        let base = RpcContextConfig::load(cfg_path).await?;
+        tracing::info!("Loaded Config");
+        let tor_proxy = base.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new(
+            Ipv4Addr::new(127, 0, 0, 1),
+            9050,
+        )));
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        let secret_store = base.secret_store().await?;
+        tracing::info!("Opened Pg DB");
+        let account = AccountInfo::load(&secret_store).await?;
+        let db = base.db(&account).await?;
+        tracing::info!("Opened PatchDB");
+        let net_controller = Arc::new(
+            NetController::init(
+                base.tor_control
+                    .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
+                tor_proxy,
+                base.dns_bind
+                    .as_deref()
+                    .unwrap_or(&[SocketAddr::from(([127, 0, 0, 1], 53))]),
+                SslManager::new(&account, root_ca_start_time().await?)?,
+                &account.hostname,
+                &account.key,
+            )
+            .await?,
+        );
+        tracing::info!("Initialized Net Controller");
+        let managers = ManagerMap::default();
+        let metrics_cache = RwLock::<Option<crate::system::Metrics>>::new(None);
+        let notification_manager = NotificationManager::new(secret_store.clone());
+        tracing::info!("Initialized Notification Manager");
+        let tor_proxy_url = format!("socks5h://{tor_proxy}");
+        let devices = lshw().await?;
+        let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
+        let seed = Arc::new(RpcContextSeed {
+            is_closed: AtomicBool::new(false),
+            datadir: base.datadir().to_path_buf(),
+            os_partitions: base.os_partitions,
+            wifi_interface: base.wifi_interface.clone(),
+            ethernet_interface: base.ethernet_interface,
+            disk_guid,
+            db,
+            secret_store,
+            account: RwLock::new(account),
+            net_controller,
+            managers,
+            metrics_cache,
+            shutdown,
+            tor_socks: tor_proxy,
+            notification_manager,
+            open_authed_websockets: Mutex::new(BTreeMap::new()),
+            rpc_stream_continuations: Mutex::new(BTreeMap::new()),
+            wifi_manager: base
+                .wifi_interface
+                .map(|i| Arc::new(RwLock::new(WpaCli::init(i)))),
+            current_secret: Arc::new(
+                Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).map_err(|e| {
+                    tracing::debug!("{:?}", e);
+                    tracing::error!("Couldn't generate ec key");
+                    Error::new(
+                        color_eyre::eyre::eyre!("Couldn't generate ec key"),
+                        crate::ErrorKind::Unknown,
+                    )
+                })?,
+            ),
+            client: Client::builder()
+                .proxy(Proxy::custom(move |url| {
+                    if url.host_str().map_or(false, |h| h.ends_with(".onion")) {
+                        Some(tor_proxy_url.clone())
+                    } else {
+                        None
+                    }
+                }))
+                .build()
+                .with_kind(crate::ErrorKind::ParseUrl)?,
+            hardware: Hardware { devices, ram },
+            start_time: Instant::now(),
+        });
+
+        let res = Self(seed.clone());
+        res.cleanup_and_initialize().await?;
+        tracing::info!("Cleaned up transient states");
+        Ok(res)
+    }
+
+    #[instrument(skip_all)]
+    pub async fn shutdown(self) -> Result<(), Error> {
+        self.managers.empty().await?;
+        self.secret_store.close().await;
+        self.is_closed.store(true, Ordering::SeqCst);
+        tracing::info!("RPC Context is shutdown");
+        // TODO: shutdown http servers
+        Ok(())
+    }
+
+    #[instrument(skip(self))]
+    pub async fn cleanup_and_initialize(&self) -> Result<(), Error> {
+        self.db
+            .mutate(|f| {
+                let mut current_dependents = f
+                    .as_package_data()
+                    .keys()?
+                    .into_iter()
+                    .map(|k| (k.clone(), BTreeMap::new()))
+                    .collect::<BTreeMap<_, _>>();
+                for (package_id, package) in f.as_package_data_mut().as_entries_mut()? {
+                    for (k, v) in package
+                        .as_installed_mut()
+                        .into_iter()
+                        .flat_map(|i| i.clone().into_current_dependencies().into_entries())
+                        .flatten()
+                    {
+                        let mut entry: BTreeMap<_, _> =
+                            current_dependents.remove(&k).unwrap_or_default();
+                        entry.insert(package_id.clone(), v.de()?);
+                        current_dependents.insert(k, entry);
+                    }
+                }
+                for (package_id, current_dependents) in current_dependents {
+                    if let Some(deps) = f
+                        .as_package_data_mut()
+                        .as_idx_mut(&package_id)
+                        .and_then(|pde| pde.expect_as_installed_mut().ok())
+                        .map(|i| i.as_installed_mut().as_current_dependents_mut())
+                    {
+                        deps.ser(&CurrentDependents(current_dependents))?;
+                    } else if let Some(deps) = f
+                        .as_package_data_mut()
+                        .as_idx_mut(&package_id)
+                        .and_then(|pde| pde.expect_as_removing_mut().ok())
+                        .map(|i| i.as_removing_mut().as_current_dependents_mut())
+                    {
+                        deps.ser(&CurrentDependents(current_dependents))?;
+                    }
+                }
+                Ok(())
+            })
+            .await?;
+
+        let peek = self.db.peek().await;
+
+        for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
+            let action = match package.as_match() {
+                PackageDataEntryMatchModelRef::Installing(_)
+                | PackageDataEntryMatchModelRef::Restoring(_)
+                | PackageDataEntryMatchModelRef::Updating(_) => {
+                    cleanup_failed(self, &package_id).await
+                }
+                PackageDataEntryMatchModelRef::Removing(_) => {
+                    uninstall(
+                        self,
+                        self.secret_store.acquire().await?.as_mut(),
+                        &package_id,
+                    )
+                    .await
+                }
+                PackageDataEntryMatchModelRef::Installed(m) => {
+                    let version = m.as_manifest().as_version().clone().de()?;
+                    let volumes = m.as_manifest().as_volumes().de()?;
+                    for (volume_id, volume_info) in &*volumes {
+                        let tmp_path = to_tmp_path(volume_info.path_for(
+                            &self.datadir,
+                            &package_id,
+                            &version,
+                            volume_id,
+                        ))
+                        .with_kind(ErrorKind::Filesystem)?;
+                        if tokio::fs::metadata(&tmp_path).await.is_ok() {
+                            tokio::fs::remove_dir_all(&tmp_path).await?;
+                        }
+                    }
+                    Ok(())
+                }
+                _ => continue,
+            };
+            if let Err(e) = action {
+                tracing::error!("Failed to clean up package {}: {}", package_id, e);
+                tracing::debug!("{:?}", e);
+            }
+        }
+        let peek = self
+            .db
+            .mutate(|v| {
+                for (_, pde) in v.as_package_data_mut().as_entries_mut()? {
+                    let status = pde
+                        .expect_as_installed_mut()?
+                        .as_installed_mut()
+                        .as_status_mut()
+                        .as_main_mut();
+                    let running = status.clone().de()?.running();
+                    status.ser(&if running {
+                        MainStatus::Starting
+                    } else {
+                        MainStatus::Stopped
+                    })?;
+                }
+                Ok(v.clone())
+            })
+            .await?;
+        self.managers.init(self.clone(), peek.clone()).await?;
+        tracing::info!("Initialized Package Managers");
+
+        let mut all_dependency_config_errs = BTreeMap::new();
+        for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
+            let package = package.clone();
+            if let Some(current_dependencies) = package
+                .as_installed()
+                .and_then(|x| x.as_current_dependencies().de().ok())
+            {
+                let manifest = package.as_manifest().de()?;
+                all_dependency_config_errs.insert(
+                    package_id.clone(),
+                    compute_dependency_config_errs(
+                        self,
+                        &peek,
+                        &manifest,
+                        &current_dependencies,
+                        &Default::default(),
+                    )
+                    .await?,
+                );
+            }
+        }
+        self.db
+            .mutate(|v| {
+                for (package_id, errs) in all_dependency_config_errs {
+                    if let Some(config_errors) = v
+                        .as_package_data_mut()
+                        .as_idx_mut(&package_id)
+                        .and_then(|pde| pde.as_installed_mut())
+                        .map(|i| i.as_status_mut().as_dependency_config_errors_mut())
+                    {
+                        config_errors.ser(&errs)?;
+                    }
+                }
+                Ok(())
+            })
+            .await?;
+
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn clean_continuations(&self) {
+        let mut continuations = self.rpc_stream_continuations.lock().await;
+        let mut to_remove = Vec::new();
+        for (guid, cont) in &*continuations {
+            if cont.is_timed_out() {
+                to_remove.push(guid.clone());
+            }
+        }
+        for guid in to_remove {
+            continuations.remove(&guid);
+        }
+    }
+
+    #[instrument(skip_all)]
+    pub async fn add_continuation(&self, guid: RequestGuid, handler: RpcContinuation) {
+        self.clean_continuations().await;
+        self.rpc_stream_continuations
+            .lock()
+            .await
+            .insert(guid, handler);
+    }
+
+    pub async fn get_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let mut continuations = self.rpc_stream_continuations.lock().await;
+        if let Some(cont) = continuations.remove(guid) {
+            cont.into_handler().await
+        } else {
+            None
+        }
+    }
+
+    pub async fn get_ws_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let continuations = self.rpc_stream_continuations.lock().await;
+        if matches!(continuations.get(guid), Some(RpcContinuation::WebSocket(_))) {
+            drop(continuations);
+            self.get_continuation_handler(guid).await
+        } else {
+            None
+        }
+    }
+
+    pub async fn get_rest_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let continuations = self.rpc_stream_continuations.lock().await;
+        if matches!(continuations.get(guid), Some(RpcContinuation::Rest(_))) {
+            drop(continuations);
+            self.get_continuation_handler(guid).await
+        } else {
+            None
+        }
+    }
+}
+impl AsRef<Jwk> for RpcContext {
+    fn as_ref(&self) -> &Jwk {
+        &CURRENT_SECRET
+    }
+}
+impl Context for RpcContext {}
+impl Deref for RpcContext {
+    type Target = RpcContextSeed;
+    fn deref(&self) -> &Self::Target {
+        #[cfg(feature = "unstable")]
+        if self.0.is_closed.load(Ordering::SeqCst) {
+            panic!(
+                "RpcContext used after shutdown! {}",
+                tracing_error::SpanTrace::capture()
+            );
+        }
+        &self.0
+    }
+}
+impl Drop for RpcContext {
+    fn drop(&mut self) {
+        #[cfg(feature = "unstable")]
+        if self.0.is_closed.load(Ordering::SeqCst) {
+            tracing::info!(
+                "RpcContext dropped. {} left.",
+                Arc::strong_count(&self.0) - 1
+            );
+        }
+    }
+}

core/startos/src/context/sdk.rs

@@ -0,0 +1,76 @@
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tracing::instrument;
+
+use crate::prelude::*;
+use crate::util::config::{load_config_from_paths, local_config_path};
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SdkContextConfig {
+    pub developer_key_path: Option<PathBuf>,
+}
+
+#[derive(Debug)]
+pub struct SdkContextSeed {
+    pub developer_key_path: PathBuf,
+}
+
+#[derive(Debug, Clone)]
+pub struct SdkContext(Arc<SdkContextSeed>);
+impl SdkContext {
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
+        let local_config_path = local_config_path();
+        let base: SdkContextConfig = load_config_from_paths(
+            matches
+                .values_of("config")
+                .into_iter()
+                .flatten()
+                .map(|p| Path::new(p))
+                .chain(local_config_path.as_deref().into_iter())
+                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+        )?;
+        Ok(SdkContext(Arc::new(SdkContextSeed {
+            developer_key_path: base.developer_key_path.unwrap_or_else(|| {
+                local_config_path
+                    .as_deref()
+                    .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
+                    .parent()
+                    .unwrap_or(Path::new("/"))
+                    .join("developer.key.pem")
+            }),
+        })))
+    }
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub fn developer_key(&self) -> Result<ed25519_dalek::SigningKey, Error> {
+        if !self.developer_key_path.exists() {
+            return Err(Error::new(eyre!("Developer Key does not exist! Please run `start-sdk init` before running this command."), crate::ErrorKind::Uninitialized));
+        }
+        let pair = <ed25519::KeypairBytes as ed25519::pkcs8::DecodePrivateKey>::from_pkcs8_pem(
+            &std::fs::read_to_string(&self.developer_key_path)?,
+        )
+        .with_kind(crate::ErrorKind::Pem)?;
+        let secret = ed25519_dalek::SecretKey::try_from(&pair.secret_key[..]).map_err(|_| {
+            Error::new(
+                eyre!("pkcs8 key is of incorrect length"),
+                ErrorKind::OpenSsl,
+            )
+        })?;
+        Ok(secret.into())
+    }
+}
+impl std::ops::Deref for SdkContext {
+    type Target = SdkContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl Context for SdkContext {}

core/startos/src/context/setup.rs

@@ -0,0 +1,149 @@
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use josekit::jwk::Jwk;
+use patch_db::json_ptr::JsonPointer;
+use patch_db::PatchDb;
+use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::Context;
+use serde::{Deserialize, Serialize};
+use sqlx::postgres::PgConnectOptions;
+use sqlx::PgPool;
+use tokio::sync::broadcast::Sender;
+use tokio::sync::RwLock;
+use tracing::instrument;
+
+use crate::account::AccountInfo;
+use crate::db::model::Database;
+use crate::disk::OsPartitionInfo;
+use crate::init::init_postgres;
+use crate::setup::SetupStatus;
+use crate::util::config::load_config_from_paths;
+use crate::{Error, ResultExt};
+
+lazy_static::lazy_static! {
+    pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| {
+        tracing::debug!("{:?}", e);
+        tracing::error!("Couldn't generate ec key");
+        panic!("Couldn't generate ec key")
+    });
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SetupResult {
+    pub tor_address: String,
+    pub lan_address: String,
+    pub root_ca: String,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SetupContextConfig {
+    pub os_partitions: OsPartitionInfo,
+    pub migration_batch_rows: Option<usize>,
+    pub migration_prefetch_rows: Option<usize>,
+    pub datadir: Option<PathBuf>,
+    #[serde(default)]
+    pub disable_encryption: bool,
+}
+impl SetupContextConfig {
+    #[instrument(skip_all)]
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+    pub fn datadir(&self) -> &Path {
+        self.datadir
+            .as_deref()
+            .unwrap_or_else(|| Path::new("/embassy-data"))
+    }
+}
+
+pub struct SetupContextSeed {
+    pub os_partitions: OsPartitionInfo,
+    pub config_path: Option<PathBuf>,
+    pub migration_batch_rows: usize,
+    pub migration_prefetch_rows: usize,
+    pub disable_encryption: bool,
+    pub shutdown: Sender<()>,
+    pub datadir: PathBuf,
+    pub selected_v2_drive: RwLock<Option<PathBuf>>,
+    pub cached_product_key: RwLock<Option<Arc<String>>>,
+    pub setup_status: RwLock<Option<Result<SetupStatus, RpcError>>>,
+    pub setup_result: RwLock<Option<(Arc<String>, SetupResult)>>,
+}
+
+impl AsRef<Jwk> for SetupContextSeed {
+    fn as_ref(&self) -> &Jwk {
+        &*CURRENT_SECRET
+    }
+}
+
+#[derive(Clone)]
+pub struct SetupContext(Arc<SetupContextSeed>);
+impl SetupContext {
+    #[instrument(skip_all)]
+    pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        let cfg = SetupContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        let datadir = cfg.datadir().to_owned();
+        Ok(Self(Arc::new(SetupContextSeed {
+            os_partitions: cfg.os_partitions,
+            config_path: path.as_ref().map(|p| p.as_ref().to_owned()),
+            migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000),
+            migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000),
+            disable_encryption: cfg.disable_encryption,
+            shutdown,
+            datadir,
+            selected_v2_drive: RwLock::new(None),
+            cached_product_key: RwLock::new(None),
+            setup_status: RwLock::new(None),
+            setup_result: RwLock::new(None),
+        })))
+    }
+    #[instrument(skip_all)]
+    pub async fn db(&self, account: &AccountInfo) -> Result<PatchDb, Error> {
+        let db_path = self.datadir.join("main").join("embassy.db");
+        let db = PatchDb::open(&db_path)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
+        if !db.exists(&<JsonPointer>::default()).await {
+            db.put(&<JsonPointer>::default(), &Database::init(account))
+                .await?;
+        }
+        Ok(db)
+    }
+    #[instrument(skip_all)]
+    pub async fn secret_store(&self) -> Result<PgPool, Error> {
+        init_postgres(&self.datadir).await?;
+        let secret_store =
+            PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
+                .await?;
+        sqlx::migrate!()
+            .run(&secret_store)
+            .await
+            .with_kind(crate::ErrorKind::Database)?;
+        Ok(secret_store)
+    }
+}
+
+impl Context for SetupContext {}
+impl Deref for SetupContext {
+    type Target = SetupContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}

core/startos/src/control.rs

@@ -0,0 +1,92 @@
+use color_eyre::eyre::eyre;
+use rpc_toolkit::command;
+use tracing::instrument;
+
+use crate::context::RpcContext;
+use crate::prelude::*;
+use crate::s9pk::manifest::PackageId;
+use crate::status::MainStatus;
+use crate::util::display_none;
+use crate::Error;
+
+#[command(display(display_none), metadata(sync_db = true))]
+#[instrument(skip_all)]
+pub async fn start(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
+    let peek = ctx.db.peek().await;
+    let version = peek
+        .as_package_data()
+        .as_idx(&id)
+        .or_not_found(&id)?
+        .as_installed()
+        .or_not_found(&id)?
+        .as_manifest()
+        .as_version()
+        .de()?;
+
+    ctx.managers
+        .get(&(id, version))
+        .await
+        .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
+        .start()
+        .await;
+
+    Ok(())
+}
+
+#[command(display(display_none), metadata(sync_db = true))]
+pub async fn stop(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<MainStatus, Error> {
+    let peek = ctx.db.peek().await;
+    let version = peek
+        .as_package_data()
+        .as_idx(&id)
+        .or_not_found(&id)?
+        .as_installed()
+        .or_not_found(&id)?
+        .as_manifest()
+        .as_version()
+        .de()?;
+
+    let last_statuts = ctx
+        .db
+        .mutate(|v| {
+            v.as_package_data_mut()
+                .as_idx_mut(&id)
+                .and_then(|x| x.as_installed_mut())
+                .ok_or_else(|| Error::new(eyre!("{} is not installed", id), ErrorKind::NotFound))?
+                .as_status_mut()
+                .as_main_mut()
+                .replace(&MainStatus::Stopping)
+        })
+        .await?;
+
+    ctx.managers
+        .get(&(id, version))
+        .await
+        .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
+        .stop()
+        .await;
+
+    Ok(last_statuts)
+}
+
+#[command(display(display_none), metadata(sync_db = true))]
+pub async fn restart(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
+    let peek = ctx.db.peek().await;
+    let version = peek
+        .as_package_data()
+        .as_idx(&id)
+        .or_not_found(&id)?
+        .expect_as_installed()?
+        .as_manifest()
+        .as_version()
+        .de()?;
+
+    ctx.managers
+        .get(&(id, version))
+        .await
+        .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
+        .restart()
+        .await;
+
+    Ok(())
+}

core/startos/src/core/mod.rs

@@ -0,0 +1 @@
+pub mod rpc_continuations;

core/startos/src/core/rpc_continuations.rs

@@ -0,0 +1,116 @@
+use std::sync::Arc;
+use std::time::Duration;
+
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use helpers::TimedResource;
+use hyper::upgrade::Upgraded;
+use hyper::{Body, Error as HyperError, Request, Response};
+use rand::RngCore;
+use tokio::task::JoinError;
+use tokio_tungstenite::WebSocketStream;
+
+use crate::{Error, ResultExt};
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
+pub struct RequestGuid<T: AsRef<str> = String>(Arc<T>);
+impl RequestGuid {
+    pub fn new() -> Self {
+        let mut buf = [0; 40];
+        rand::thread_rng().fill_bytes(&mut buf);
+        RequestGuid(Arc::new(base32::encode(
+            base32::Alphabet::RFC4648 { padding: false },
+            &buf,
+        )))
+    }
+
+    pub fn from(r: &str) -> Option<RequestGuid> {
+        if r.len() != 64 {
+            return None;
+        }
+        for c in r.chars() {
+            if !(c >= 'A' && c <= 'Z' || c >= '2' && c <= '7') {
+                return None;
+            }
+        }
+        Some(RequestGuid(Arc::new(r.to_owned())))
+    }
+}
+#[test]
+fn parse_guid() {
+    println!(
+        "{:?}",
+        RequestGuid::from(&format!("{}", RequestGuid::new()))
+    )
+}
+
+impl<T: AsRef<str>> std::fmt::Display for RequestGuid<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        (&*self.0).as_ref().fmt(f)
+    }
+}
+
+pub type RestHandler = Box<
+    dyn FnOnce(Request<Body>) -> BoxFuture<'static, Result<Response<Body>, crate::Error>> + Send,
+>;
+
+pub type WebSocketHandler = Box<
+    dyn FnOnce(
+            BoxFuture<'static, Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
+        ) -> BoxFuture<'static, Result<(), Error>>
+        + Send,
+>;
+
+pub enum RpcContinuation {
+    Rest(TimedResource<RestHandler>),
+    WebSocket(TimedResource<WebSocketHandler>),
+}
+impl RpcContinuation {
+    pub fn rest(handler: RestHandler, timeout: Duration) -> Self {
+        RpcContinuation::Rest(TimedResource::new(handler, timeout))
+    }
+    pub fn ws(handler: WebSocketHandler, timeout: Duration) -> Self {
+        RpcContinuation::WebSocket(TimedResource::new(handler, timeout))
+    }
+    pub fn is_timed_out(&self) -> bool {
+        match self {
+            RpcContinuation::Rest(a) => a.is_timed_out(),
+            RpcContinuation::WebSocket(a) => a.is_timed_out(),
+        }
+    }
+    pub async fn into_handler(self) -> Option<RestHandler> {
+        match self {
+            RpcContinuation::Rest(handler) => handler.get().await,
+            RpcContinuation::WebSocket(handler) => {
+                if let Some(handler) = handler.get().await {
+                    Some(Box::new(
+                        |req: Request<Body>| -> BoxFuture<'static, Result<Response<Body>, Error>> {
+                            async move {
+                                let (parts, body) = req.into_parts();
+                                let req = Request::from_parts(parts, body);
+                                let (res, ws_fut) = hyper_ws_listener::create_ws(req)
+                                    .with_kind(crate::ErrorKind::Network)?;
+                                if let Some(ws_fut) = ws_fut {
+                                    tokio::task::spawn(async move {
+                                        match handler(ws_fut.boxed()).await {
+                                            Ok(()) => (),
+                                            Err(e) => {
+                                                tracing::error!("WebSocket Closed: {}", e);
+                                                tracing::debug!("{:?}", e);
+                                            }
+                                        }
+                                    });
+                                }
+
+                                Ok(res)
+                            }
+                            .boxed()
+                        },
+                    ))
+                } else {
+                    None
+                }
+            }
+        }
+    }
+}

core/startos/src/db/mod.rs

@@ -0,0 +1,370 @@
+pub mod model;
+pub mod package;
+pub mod prelude;
+
+use std::future::Future;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use futures::{FutureExt, SinkExt, StreamExt};
+use patch_db::json_ptr::JsonPointer;
+use patch_db::{Dump, Revision};
+use rpc_toolkit::command;
+use rpc_toolkit::hyper::upgrade::Upgraded;
+use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response};
+use rpc_toolkit::yajrc::RpcError;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use tokio::sync::oneshot;
+use tokio::task::JoinError;
+use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
+use tokio_tungstenite::tungstenite::protocol::CloseFrame;
+use tokio_tungstenite::tungstenite::Message;
+use tokio_tungstenite::WebSocketStream;
+use tracing::instrument;
+
+use crate::context::{CliContext, RpcContext};
+use crate::middleware::auth::{HasValidSession, HashSessionToken};
+use crate::prelude::*;
+use crate::util::display_none;
+use crate::util::serde::{display_serializable, IoFormat};
+
+#[instrument(skip_all)]
+async fn ws_handler<
+    WSFut: Future<Output = Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
+>(
+    ctx: RpcContext,
+    session: Option<(HasValidSession, HashSessionToken)>,
+    ws_fut: WSFut,
+) -> Result<(), Error> {
+    let (dump, sub) = ctx.db.dump_and_sub().await;
+    let mut stream = ws_fut
+        .await
+        .with_kind(ErrorKind::Network)?
+        .with_kind(ErrorKind::Unknown)?;
+
+    if let Some((session, token)) = session {
+        let kill = subscribe_to_session_kill(&ctx, token).await;
+        send_dump(session, &mut stream, dump).await?;
+
+        deal_with_messages(session, kill, sub, stream).await?;
+    } else {
+        stream
+            .close(Some(CloseFrame {
+                code: CloseCode::Error,
+                reason: "UNAUTHORIZED".into(),
+            }))
+            .await
+            .with_kind(ErrorKind::Network)?;
+    }
+
+    Ok(())
+}
+
+async fn subscribe_to_session_kill(
+    ctx: &RpcContext,
+    token: HashSessionToken,
+) -> oneshot::Receiver<()> {
+    let (send, recv) = oneshot::channel();
+    let mut guard = ctx.open_authed_websockets.lock().await;
+    if !guard.contains_key(&token) {
+        guard.insert(token, vec![send]);
+    } else {
+        guard.get_mut(&token).unwrap().push(send);
+    }
+    recv
+}
+
+#[instrument(skip_all)]
+async fn deal_with_messages(
+    _has_valid_authentication: HasValidSession,
+    mut kill: oneshot::Receiver<()>,
+    mut sub: patch_db::Subscriber,
+    mut stream: WebSocketStream<Upgraded>,
+) -> Result<(), Error> {
+    let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(5));
+
+    loop {
+        futures::select! {
+            _ = (&mut kill).fuse() => {
+                tracing::info!("Closing WebSocket: Reason: Session Terminated");
+                stream
+                    .close(Some(CloseFrame {
+                        code: CloseCode::Error,
+                        reason: "UNAUTHORIZED".into(),
+                    }))
+                    .await
+                    .with_kind(ErrorKind::Network)?;
+                return Ok(())
+            }
+            new_rev = sub.recv().fuse() => {
+                let rev = new_rev.expect("UNREACHABLE: patch-db is dropped");
+                stream
+                    .send(Message::Text(serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?))
+                    .await
+                    .with_kind(ErrorKind::Network)?;
+            }
+            message = stream.next().fuse() => {
+                let message = message.transpose().with_kind(ErrorKind::Network)?;
+                match message {
+                    None => {
+                        tracing::info!("Closing WebSocket: Stream Finished");
+                        return Ok(())
+                    }
+                    _ => (),
+                }
+            }
+            // This is trying to give a health checks to the home to keep the ui alive.
+            _ = timer.tick().fuse() => {
+                stream
+                    .send(Message::Ping(vec![]))
+                    .await
+                    .with_kind(crate::ErrorKind::Network)?;
+            }
+        }
+    }
+}
+
+async fn send_dump(
+    _has_valid_authentication: HasValidSession,
+    stream: &mut WebSocketStream<Upgraded>,
+    dump: Dump,
+) -> Result<(), Error> {
+    stream
+        .send(Message::Text(
+            serde_json::to_string(&dump).with_kind(ErrorKind::Serialization)?,
+        ))
+        .await
+        .with_kind(ErrorKind::Network)?;
+    Ok(())
+}
+
+pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<Body>, Error> {
+    let (parts, body) = req.into_parts();
+    let session = match async {
+        let token = HashSessionToken::from_request_parts(&parts)?;
+        let session = HasValidSession::from_request_parts(&parts, &ctx).await?;
+        Ok::<_, Error>((session, token))
+    }
+    .await
+    {
+        Ok(a) => Some(a),
+        Err(e) => {
+            if e.kind != ErrorKind::Authorization {
+                tracing::error!("Error Authenticating Websocket: {}", e);
+                tracing::debug!("{:?}", e);
+            }
+            None
+        }
+    };
+    let req = Request::from_parts(parts, body);
+    let (res, ws_fut) = hyper_ws_listener::create_ws(req).with_kind(ErrorKind::Network)?;
+    if let Some(ws_fut) = ws_fut {
+        tokio::task::spawn(async move {
+            match ws_handler(ctx, session, ws_fut).await {
+                Ok(()) => (),
+                Err(e) => {
+                    tracing::error!("WebSocket Closed: {}", e);
+                    tracing::debug!("{:?}", e);
+                }
+            }
+        });
+    }
+
+    Ok(res)
+}
+
+#[command(subcommands(dump, put, apply))]
+pub fn db() -> Result<(), RpcError> {
+    Ok(())
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(untagged)]
+pub enum RevisionsRes {
+    Revisions(Vec<Arc<Revision>>),
+    Dump(Dump),
+}
+
+#[instrument(skip_all)]
+async fn cli_dump(
+    ctx: CliContext,
+    _format: Option<IoFormat>,
+    path: Option<PathBuf>,
+) -> Result<Dump, RpcError> {
+    let dump = if let Some(path) = path {
+        PatchDb::open(path).await?.dump().await
+    } else {
+        rpc_toolkit::command_helpers::call_remote(
+            ctx,
+            "db.dump",
+            serde_json::json!({}),
+            std::marker::PhantomData::<Dump>,
+        )
+        .await?
+        .result?
+    };
+
+    Ok(dump)
+}
+
+#[command(
+    custom_cli(cli_dump(async, context(CliContext))),
+    display(display_serializable)
+)]
+pub async fn dump(
+    #[context] ctx: RpcContext,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+    #[allow(unused_variables)]
+    #[arg]
+    path: Option<PathBuf>,
+) -> Result<Dump, Error> {
+    Ok(ctx.db.dump().await)
+}
+
+fn apply_expr(input: jaq_core::Val, expr: &str) -> Result<jaq_core::Val, Error> {
+    let (expr, errs) = jaq_core::parse::parse(expr, jaq_core::parse::main());
+
+    let Some(expr) = expr else {
+        return Err(Error::new(
+            eyre!("Failed to parse expression: {:?}", errs),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    let mut errs = Vec::new();
+
+    let mut defs = jaq_core::Definitions::core();
+    for def in jaq_std::std() {
+        defs.insert(def, &mut errs);
+    }
+
+    let filter = defs.finish(expr, Vec::new(), &mut errs);
+
+    if !errs.is_empty() {
+        return Err(Error::new(
+            eyre!("Failed to compile expression: {:?}", errs),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    let inputs = jaq_core::RcIter::new(std::iter::empty());
+    let mut res_iter = filter.run(jaq_core::Ctx::new([], &inputs), input);
+
+    let Some(res) = res_iter
+        .next()
+        .transpose()
+        .map_err(|e| eyre!("{e}"))
+        .with_kind(crate::ErrorKind::Deserialization)?
+    else {
+        return Err(Error::new(
+            eyre!("expr returned no results"),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    if res_iter.next().is_some() {
+        return Err(Error::new(
+            eyre!("expr returned too many results"),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    }
+
+    Ok(res)
+}
+
+#[instrument(skip_all)]
+async fn cli_apply(ctx: CliContext, expr: String, path: Option<PathBuf>) -> Result<(), RpcError> {
+    if let Some(path) = path {
+        PatchDb::open(path)
+            .await?
+            .mutate(|db| {
+                let res = apply_expr(
+                    serde_json::to_value(patch_db::Value::from(db.clone()))
+                        .with_kind(ErrorKind::Deserialization)?
+                        .into(),
+                    &expr,
+                )?;
+
+                db.ser(
+                    &serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(
+                        |_| {
+                            (
+                                crate::ErrorKind::Deserialization,
+                                "result does not match database model",
+                            )
+                        },
+                    )?,
+                )
+            })
+            .await?;
+    } else {
+        rpc_toolkit::command_helpers::call_remote(
+            ctx,
+            "db.apply",
+            serde_json::json!({ "expr": expr }),
+            std::marker::PhantomData::<()>,
+        )
+        .await?
+        .result?;
+    }
+
+    Ok(())
+}
+
+#[command(
+    custom_cli(cli_apply(async, context(CliContext))),
+    display(display_none)
+)]
+pub async fn apply(
+    #[context] ctx: RpcContext,
+    #[arg] expr: String,
+    #[allow(unused_variables)]
+    #[arg]
+    path: Option<PathBuf>,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            let res = apply_expr(
+                serde_json::to_value(patch_db::Value::from(db.clone()))
+                    .with_kind(ErrorKind::Deserialization)?
+                    .into(),
+                &expr,
+            )?;
+
+            db.ser(
+                &serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Deserialization,
+                        "result does not match database model",
+                    )
+                })?,
+            )
+        })
+        .await
+}
+
+#[command(subcommands(ui))]
+pub fn put() -> Result<(), RpcError> {
+    Ok(())
+}
+
+#[command(display(display_serializable))]
+#[instrument(skip_all)]
+pub async fn ui(
+    #[context] ctx: RpcContext,
+    #[arg] pointer: JsonPointer,
+    #[arg] value: Value,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+) -> Result<(), Error> {
+    let ptr = "/ui"
+        .parse::<JsonPointer>()
+        .with_kind(ErrorKind::Database)?
+        + &pointer;
+    ctx.db.put(&ptr, &value).await?;
+    Ok(())
+}

core/startos/src/db/model.rs

@@ -0,0 +1,527 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::net::{Ipv4Addr, Ipv6Addr};
+use std::sync::Arc;
+
+use chrono::{DateTime, Utc};
+use emver::VersionRange;
+use imbl_value::InternedString;
+use ipnet::{Ipv4Net, Ipv6Net};
+use isocountry::CountryCode;
+use itertools::Itertools;
+use models::{DataUrl, HealthCheckId, InterfaceId};
+use openssl::hash::MessageDigest;
+use patch_db::{HasModel, Value};
+use reqwest::Url;
+use serde::{Deserialize, Serialize};
+use ssh_key::public::Ed25519PublicKey;
+
+use crate::account::AccountInfo;
+use crate::config::spec::PackagePointerSpec;
+use crate::install::progress::InstallProgress;
+use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
+use crate::prelude::*;
+use crate::s9pk::manifest::{Manifest, PackageId};
+use crate::status::Status;
+use crate::util::Version;
+use crate::version::{Current, VersionT};
+use crate::{ARCH, PLATFORM};
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+// #[macro_debug]
+pub struct Database {
+    pub server_info: ServerInfo,
+    pub package_data: AllPackageData,
+    pub ui: Value,
+}
+impl Database {
+    pub fn init(account: &AccountInfo) -> Self {
+        let lan_address = account.hostname.lan_address().parse().unwrap();
+        Database {
+            server_info: ServerInfo {
+                arch: get_arch(),
+                platform: get_platform(),
+                id: account.server_id.clone(),
+                version: Current::new().semver().into(),
+                hostname: account.hostname.no_dot_host_name(),
+                last_backup: None,
+                last_wifi_region: None,
+                eos_version_compat: Current::new().compat().clone(),
+                lan_address,
+                tor_address: format!("https://{}", account.key.tor_address())
+                    .parse()
+                    .unwrap(),
+                ip_info: BTreeMap::new(),
+                status_info: ServerStatus {
+                    backup_progress: None,
+                    updated: false,
+                    update_progress: None,
+                    shutting_down: false,
+                    restarting: false,
+                },
+                wifi: WifiInfo {
+                    ssids: Vec::new(),
+                    connected: None,
+                    selected: None,
+                },
+                unread_notification_count: 0,
+                connection_addresses: ConnectionAddresses {
+                    tor: Vec::new(),
+                    clearnet: Vec::new(),
+                },
+                password_hash: account.password.clone(),
+                pubkey: ssh_key::PublicKey::from(Ed25519PublicKey::from(&account.key.ssh_key()))
+                    .to_openssh()
+                    .unwrap(),
+                ca_fingerprint: account
+                    .root_ca_cert
+                    .digest(MessageDigest::sha256())
+                    .unwrap()
+                    .iter()
+                    .map(|x| format!("{x:X}"))
+                    .join(":"),
+                ntp_synced: false,
+                zram: true,
+            },
+            package_data: AllPackageData::default(),
+            ui: serde_json::from_str(include_str!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../../web/patchdb-ui-seed.json"
+            )))
+            .unwrap(),
+        }
+    }
+}
+
+pub type DatabaseModel = Model<Database>;
+
+fn get_arch() -> InternedString {
+    (*ARCH).into()
+}
+
+fn get_platform() -> InternedString {
+    (&*PLATFORM).into()
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct ServerInfo {
+    #[serde(default = "get_arch")]
+    pub arch: InternedString,
+    #[serde(default = "get_platform")]
+    pub platform: InternedString,
+    pub id: String,
+    pub hostname: String,
+    pub version: Version,
+    pub last_backup: Option<DateTime<Utc>>,
+    /// Used in the wifi to determine the region to set the system to
+    pub last_wifi_region: Option<CountryCode>,
+    pub eos_version_compat: VersionRange,
+    pub lan_address: Url,
+    pub tor_address: Url,
+    pub ip_info: BTreeMap<String, IpInfo>,
+    #[serde(default)]
+    pub status_info: ServerStatus,
+    pub wifi: WifiInfo,
+    pub unread_notification_count: u64,
+    pub connection_addresses: ConnectionAddresses,
+    pub password_hash: String,
+    pub pubkey: String,
+    pub ca_fingerprint: String,
+    #[serde(default)]
+    pub ntp_synced: bool,
+    #[serde(default)]
+    pub zram: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct IpInfo {
+    pub ipv4_range: Option<Ipv4Net>,
+    pub ipv4: Option<Ipv4Addr>,
+    pub ipv6_range: Option<Ipv6Net>,
+    pub ipv6: Option<Ipv6Addr>,
+}
+impl IpInfo {
+    pub async fn for_interface(iface: &str) -> Result<Self, Error> {
+        let (ipv4, ipv4_range) = get_iface_ipv4_addr(iface).await?.unzip();
+        let (ipv6, ipv6_range) = get_iface_ipv6_addr(iface).await?.unzip();
+        Ok(Self {
+            ipv4_range,
+            ipv4,
+            ipv6_range,
+            ipv6,
+        })
+    }
+}
+
+#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
+#[model = "Model<Self>"]
+pub struct BackupProgress {
+    pub complete: bool,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct ServerStatus {
+    pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
+    pub updated: bool,
+    pub update_progress: Option<UpdateProgress>,
+    #[serde(default)]
+    pub shutting_down: bool,
+    #[serde(default)]
+    pub restarting: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct UpdateProgress {
+    pub size: Option<u64>,
+    pub downloaded: u64,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct WifiInfo {
+    pub ssids: Vec<String>,
+    pub selected: Option<String>,
+    pub connected: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct ServerSpecs {
+    pub cpu: String,
+    pub disk: String,
+    pub memory: String,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct ConnectionAddresses {
+    pub tor: Vec<String>,
+    pub clearnet: Vec<String>,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct AllPackageData(pub BTreeMap<PackageId, PackageDataEntry>);
+impl Map for AllPackageData {
+    type Key = PackageId;
+    type Value = PackageDataEntry;
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct StaticFiles {
+    license: String,
+    instructions: String,
+    icon: String,
+}
+impl StaticFiles {
+    pub fn local(id: &PackageId, version: &Version, icon_type: &str) -> Self {
+        StaticFiles {
+            license: format!("/public/package-data/{}/{}/LICENSE.md", id, version),
+            instructions: format!("/public/package-data/{}/{}/INSTRUCTIONS.md", id, version),
+            icon: format!("/public/package-data/{}/{}/icon.{}", id, version, icon_type),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct PackageDataEntryInstalling {
+    pub static_files: StaticFiles,
+    pub manifest: Manifest,
+    pub install_progress: Arc<InstallProgress>,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct PackageDataEntryUpdating {
+    pub static_files: StaticFiles,
+    pub manifest: Manifest,
+    pub installed: InstalledPackageInfo,
+    pub install_progress: Arc<InstallProgress>,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct PackageDataEntryRestoring {
+    pub static_files: StaticFiles,
+    pub manifest: Manifest,
+    pub install_progress: Arc<InstallProgress>,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct PackageDataEntryRemoving {
+    pub static_files: StaticFiles,
+    pub manifest: Manifest,
+    pub removing: InstalledPackageInfo,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct PackageDataEntryInstalled {
+    pub static_files: StaticFiles,
+    pub manifest: Manifest,
+    pub installed: InstalledPackageInfo,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(tag = "state")]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+// #[macro_debug]
+pub enum PackageDataEntry {
+    Installing(PackageDataEntryInstalling),
+    Updating(PackageDataEntryUpdating),
+    Restoring(PackageDataEntryRestoring),
+    Removing(PackageDataEntryRemoving),
+    Installed(PackageDataEntryInstalled),
+}
+impl Model<PackageDataEntry> {
+    pub fn expect_into_installed(self) -> Result<Model<PackageDataEntryInstalled>, Error> {
+        if let PackageDataEntryMatchModel::Installed(a) = self.into_match() {
+            Ok(a)
+        } else {
+            Err(Error::new(
+                eyre!("package is not in installed state"),
+                ErrorKind::InvalidRequest,
+            ))
+        }
+    }
+    pub fn expect_as_installed(&self) -> Result<&Model<PackageDataEntryInstalled>, Error> {
+        if let PackageDataEntryMatchModelRef::Installed(a) = self.as_match() {
+            Ok(a)
+        } else {
+            Err(Error::new(
+                eyre!("package is not in installed state"),
+                ErrorKind::InvalidRequest,
+            ))
+        }
+    }
+    pub fn expect_as_installed_mut(
+        &mut self,
+    ) -> Result<&mut Model<PackageDataEntryInstalled>, Error> {
+        if let PackageDataEntryMatchModelMut::Installed(a) = self.as_match_mut() {
+            Ok(a)
+        } else {
+            Err(Error::new(
+                eyre!("package is not in installed state"),
+                ErrorKind::InvalidRequest,
+            ))
+        }
+    }
+    pub fn expect_into_removing(self) -> Result<Model<PackageDataEntryRemoving>, Error> {
+        if let PackageDataEntryMatchModel::Removing(a) = self.into_match() {
+            Ok(a)
+        } else {
+            Err(Error::new(
+                eyre!("package is not in removing state"),
+                ErrorKind::InvalidRequest,
+            ))
+        }
+    }
+    pub fn expect_as_removing(&self) -> Result<&Model<PackageDataEntryRemoving>, Error> {
+        if let PackageDataEntryMatchModelRef::Removing(a) = self.as_match() {
+            Ok(a)
+        } else {
+            Err(Error::new(
+                eyre!("package is not in removing state"),
+                ErrorKind::InvalidRequest,
+            ))
+        }
+    }
+    pub fn expect_as_removing_mut(
+        &mut self,
+    ) -> Result<&mut Model<PackageDataEntryRemoving>, Error> {
+        if let PackageDataEntryMatchModelMut::Removing(a) = self.as_match_mut() {
+            Ok(a)
+        } else {
+            Err(Error::new(
+                eyre!("package is not in removing state"),
+                ErrorKind::InvalidRequest,
+            ))
+        }
+    }
+    pub fn expect_as_installing_mut(
+        &mut self,
+    ) -> Result<&mut Model<PackageDataEntryInstalling>, Error> {
+        if let PackageDataEntryMatchModelMut::Installing(a) = self.as_match_mut() {
+            Ok(a)
+        } else {
+            Err(Error::new(
+                eyre!("package is not in installing state"),
+                ErrorKind::InvalidRequest,
+            ))
+        }
+    }
+    pub fn into_manifest(self) -> Model<Manifest> {
+        match self.into_match() {
+            PackageDataEntryMatchModel::Installing(a) => a.into_manifest(),
+            PackageDataEntryMatchModel::Updating(a) => a.into_installed().into_manifest(),
+            PackageDataEntryMatchModel::Restoring(a) => a.into_manifest(),
+            PackageDataEntryMatchModel::Removing(a) => a.into_manifest(),
+            PackageDataEntryMatchModel::Installed(a) => a.into_manifest(),
+            PackageDataEntryMatchModel::Error(_) => Model::from(Value::Null),
+        }
+    }
+    pub fn as_manifest(&self) -> &Model<Manifest> {
+        match self.as_match() {
+            PackageDataEntryMatchModelRef::Installing(a) => a.as_manifest(),
+            PackageDataEntryMatchModelRef::Updating(a) => a.as_installed().as_manifest(),
+            PackageDataEntryMatchModelRef::Restoring(a) => a.as_manifest(),
+            PackageDataEntryMatchModelRef::Removing(a) => a.as_manifest(),
+            PackageDataEntryMatchModelRef::Installed(a) => a.as_manifest(),
+            PackageDataEntryMatchModelRef::Error(_) => (&Value::Null).into(),
+        }
+    }
+    pub fn into_installed(self) -> Option<Model<InstalledPackageInfo>> {
+        match self.into_match() {
+            PackageDataEntryMatchModel::Installing(_) => None,
+            PackageDataEntryMatchModel::Updating(a) => Some(a.into_installed()),
+            PackageDataEntryMatchModel::Restoring(_) => None,
+            PackageDataEntryMatchModel::Removing(_) => None,
+            PackageDataEntryMatchModel::Installed(a) => Some(a.into_installed()),
+            PackageDataEntryMatchModel::Error(_) => None,
+        }
+    }
+    pub fn as_installed(&self) -> Option<&Model<InstalledPackageInfo>> {
+        match self.as_match() {
+            PackageDataEntryMatchModelRef::Installing(_) => None,
+            PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_installed()),
+            PackageDataEntryMatchModelRef::Restoring(_) => None,
+            PackageDataEntryMatchModelRef::Removing(_) => None,
+            PackageDataEntryMatchModelRef::Installed(a) => Some(a.as_installed()),
+            PackageDataEntryMatchModelRef::Error(_) => None,
+        }
+    }
+    pub fn as_installed_mut(&mut self) -> Option<&mut Model<InstalledPackageInfo>> {
+        match self.as_match_mut() {
+            PackageDataEntryMatchModelMut::Installing(_) => None,
+            PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_installed_mut()),
+            PackageDataEntryMatchModelMut::Restoring(_) => None,
+            PackageDataEntryMatchModelMut::Removing(_) => None,
+            PackageDataEntryMatchModelMut::Installed(a) => Some(a.as_installed_mut()),
+            PackageDataEntryMatchModelMut::Error(_) => None,
+        }
+    }
+    pub fn as_install_progress(&self) -> Option<&Model<Arc<InstallProgress>>> {
+        match self.as_match() {
+            PackageDataEntryMatchModelRef::Installing(a) => Some(a.as_install_progress()),
+            PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_install_progress()),
+            PackageDataEntryMatchModelRef::Restoring(a) => Some(a.as_install_progress()),
+            PackageDataEntryMatchModelRef::Removing(_) => None,
+            PackageDataEntryMatchModelRef::Installed(_) => None,
+            PackageDataEntryMatchModelRef::Error(_) => None,
+        }
+    }
+    pub fn as_install_progress_mut(&mut self) -> Option<&mut Model<Arc<InstallProgress>>> {
+        match self.as_match_mut() {
+            PackageDataEntryMatchModelMut::Installing(a) => Some(a.as_install_progress_mut()),
+            PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_install_progress_mut()),
+            PackageDataEntryMatchModelMut::Restoring(a) => Some(a.as_install_progress_mut()),
+            PackageDataEntryMatchModelMut::Removing(_) => None,
+            PackageDataEntryMatchModelMut::Installed(_) => None,
+            PackageDataEntryMatchModelMut::Error(_) => None,
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct InstalledPackageInfo {
+    pub status: Status,
+    pub marketplace_url: Option<Url>,
+    #[serde(default)]
+    #[serde(with = "crate::util::serde::ed25519_pubkey")]
+    pub developer_key: ed25519_dalek::VerifyingKey,
+    pub manifest: Manifest,
+    pub last_backup: Option<DateTime<Utc>>,
+    pub dependency_info: BTreeMap<PackageId, StaticDependencyInfo>,
+    pub current_dependents: CurrentDependents,
+    pub current_dependencies: CurrentDependencies,
+    pub interface_addresses: InterfaceAddressMap,
+}
+
+#[derive(Debug, Clone, Default, Deserialize, Serialize)]
+pub struct CurrentDependents(pub BTreeMap<PackageId, CurrentDependencyInfo>);
+impl CurrentDependents {
+    pub fn map(
+        mut self,
+        transform: impl Fn(
+            BTreeMap<PackageId, CurrentDependencyInfo>,
+        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
+    ) -> Self {
+        self.0 = transform(self.0);
+        self
+    }
+}
+impl Map for CurrentDependents {
+    type Key = PackageId;
+    type Value = CurrentDependencyInfo;
+}
+
+#[derive(Debug, Clone, Default, Deserialize, Serialize)]
+pub struct CurrentDependencies(pub BTreeMap<PackageId, CurrentDependencyInfo>);
+impl CurrentDependencies {
+    pub fn map(
+        mut self,
+        transform: impl Fn(
+            BTreeMap<PackageId, CurrentDependencyInfo>,
+        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
+    ) -> Self {
+        self.0 = transform(self.0);
+        self
+    }
+}
+impl Map for CurrentDependencies {
+    type Key = PackageId;
+    type Value = CurrentDependencyInfo;
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct StaticDependencyInfo {
+    pub title: String,
+    pub icon: DataUrl<'static>,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct CurrentDependencyInfo {
+    #[serde(default)]
+    pub pointers: BTreeSet<PackagePointerSpec>,
+    pub health_checks: BTreeSet<HealthCheckId>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct InterfaceAddressMap(pub BTreeMap<InterfaceId, InterfaceAddresses>);
+impl Map for InterfaceAddressMap {
+    type Key = InterfaceId;
+    type Value = InterfaceAddresses;
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct InterfaceAddresses {
+    pub tor_address: Option<String>,
+    pub lan_address: Option<String>,
+}

core/startos/src/db/package.rs

@@ -0,0 +1,22 @@
+use models::Version;
+
+use crate::prelude::*;
+use crate::s9pk::manifest::PackageId;
+
+pub fn get_packages(db: Peeked) -> Result<Vec<(PackageId, Version)>, Error> {
+    Ok(db
+        .as_package_data()
+        .keys()?
+        .into_iter()
+        .flat_map(|package_id| {
+            let version = db
+                .as_package_data()
+                .as_idx(&package_id)?
+                .as_manifest()
+                .as_version()
+                .de()
+                .ok()?;
+            Some((package_id, version))
+        })
+        .collect())
+}

core/startos/src/db/prelude.rs

@@ -0,0 +1,382 @@
+use std::collections::BTreeMap;
+use std::marker::PhantomData;
+use std::panic::UnwindSafe;
+
+use patch_db::value::InternedString;
+pub use patch_db::{HasModel, PatchDb, Value};
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+use crate::db::model::DatabaseModel;
+use crate::prelude::*;
+
+pub type Peeked = Model<super::model::Database>;
+
+pub fn to_value<T>(value: &T) -> Result<Value, Error>
+where
+    T: Serialize,
+{
+    patch_db::value::to_value(value).with_kind(ErrorKind::Serialization)
+}
+
+pub fn from_value<T>(value: Value) -> Result<T, Error>
+where
+    T: DeserializeOwned,
+{
+    patch_db::value::from_value(value).with_kind(ErrorKind::Deserialization)
+}
+
+#[async_trait::async_trait]
+pub trait PatchDbExt {
+    async fn peek(&self) -> DatabaseModel;
+    async fn mutate<U: UnwindSafe + Send>(
+        &self,
+        f: impl FnOnce(&mut DatabaseModel) -> Result<U, Error> + UnwindSafe + Send,
+    ) -> Result<U, Error>;
+    async fn map_mutate(
+        &self,
+        f: impl FnOnce(DatabaseModel) -> Result<DatabaseModel, Error> + UnwindSafe + Send,
+    ) -> Result<DatabaseModel, Error>;
+}
+#[async_trait::async_trait]
+impl PatchDbExt for PatchDb {
+    async fn peek(&self) -> DatabaseModel {
+        DatabaseModel::from(self.dump().await.value)
+    }
+    async fn mutate<U: UnwindSafe + Send>(
+        &self,
+        f: impl FnOnce(&mut DatabaseModel) -> Result<U, Error> + UnwindSafe + Send,
+    ) -> Result<U, Error> {
+        Ok(self
+            .apply_function(|mut v| {
+                let model = <&mut DatabaseModel>::from(&mut v);
+                let res = f(model)?;
+                Ok::<_, Error>((v, res))
+            })
+            .await?
+            .1)
+    }
+    async fn map_mutate(
+        &self,
+        f: impl FnOnce(DatabaseModel) -> Result<DatabaseModel, Error> + UnwindSafe + Send,
+    ) -> Result<DatabaseModel, Error> {
+        Ok(DatabaseModel::from(
+            self.apply_function(|v| f(DatabaseModel::from(v)).map(|a| (a.into(), ())))
+                .await?
+                .0,
+        ))
+    }
+}
+
+/// &mut Model<T> <=> &mut Value
+#[repr(transparent)]
+#[derive(Debug)]
+pub struct Model<T> {
+    value: Value,
+    phantom: PhantomData<T>,
+}
+impl<T: DeserializeOwned> Model<T> {
+    pub fn de(&self) -> Result<T, Error> {
+        from_value(self.value.clone())
+    }
+}
+impl<T: Serialize> Model<T> {
+    pub fn new(value: &T) -> Result<Self, Error> {
+        Ok(Self::from(to_value(value)?))
+    }
+    pub fn ser(&mut self, value: &T) -> Result<(), Error> {
+        self.value = to_value(value)?;
+        Ok(())
+    }
+}
+
+impl<T: Serialize + DeserializeOwned> Model<T> {
+    pub fn replace(&mut self, value: &T) -> Result<T, Error> {
+        let orig = self.de()?;
+        self.ser(value)?;
+        Ok(orig)
+    }
+}
+impl<T> Clone for Model<T> {
+    fn clone(&self) -> Self {
+        Self {
+            value: self.value.clone(),
+            phantom: PhantomData,
+        }
+    }
+}
+impl<T> From<Value> for Model<T> {
+    fn from(value: Value) -> Self {
+        Self {
+            value,
+            phantom: PhantomData,
+        }
+    }
+}
+impl<T> From<Model<T>> for Value {
+    fn from(value: Model<T>) -> Self {
+        value.value
+    }
+}
+impl<'a, T> From<&'a Value> for &'a Model<T> {
+    fn from(value: &'a Value) -> Self {
+        unsafe { std::mem::transmute(value) }
+    }
+}
+impl<'a, T> From<&'a Model<T>> for &'a Value {
+    fn from(value: &'a Model<T>) -> Self {
+        unsafe { std::mem::transmute(value) }
+    }
+}
+impl<'a, T> From<&'a mut Value> for &mut Model<T> {
+    fn from(value: &'a mut Value) -> Self {
+        unsafe { std::mem::transmute(value) }
+    }
+}
+impl<'a, T> From<&'a mut Model<T>> for &mut Value {
+    fn from(value: &'a mut Model<T>) -> Self {
+        unsafe { std::mem::transmute(value) }
+    }
+}
+impl<T> patch_db::Model<T> for Model<T> {
+    type Model<U> = Model<U>;
+}
+
+impl<T> Model<Option<T>> {
+    pub fn transpose(self) -> Option<Model<T>> {
+        use patch_db::ModelExt;
+        if self.value.is_null() {
+            None
+        } else {
+            Some(self.transmute(|a| a))
+        }
+    }
+    pub fn transpose_ref(&self) -> Option<&Model<T>> {
+        use patch_db::ModelExt;
+        if self.value.is_null() {
+            None
+        } else {
+            Some(self.transmute_ref(|a| a))
+        }
+    }
+    pub fn transpose_mut(&mut self) -> Option<&mut Model<T>> {
+        use patch_db::ModelExt;
+        if self.value.is_null() {
+            None
+        } else {
+            Some(self.transmute_mut(|a| a))
+        }
+    }
+    pub fn from_option(opt: Option<Model<T>>) -> Self {
+        use patch_db::ModelExt;
+        match opt {
+            Some(a) => a.transmute(|a| a),
+            None => Self::from_value(Value::Null),
+        }
+    }
+}
+
+pub trait Map: DeserializeOwned + Serialize {
+    type Key;
+    type Value;
+}
+
+impl<A, B> Map for BTreeMap<A, B>
+where
+    A: serde::Serialize + serde::de::DeserializeOwned + Ord,
+    B: serde::Serialize + serde::de::DeserializeOwned,
+{
+    type Key = A;
+    type Value = B;
+}
+
+impl<T: Map> Model<T>
+where
+    T::Key: AsRef<str>,
+    T::Value: Serialize,
+{
+    pub fn insert(&mut self, key: &T::Key, value: &T::Value) -> Result<(), Error> {
+        use serde::ser::Error;
+        let v = patch_db::value::to_value(value)?;
+        match &mut self.value {
+            Value::Object(o) => {
+                o.insert(InternedString::intern(key.as_ref()), v);
+                Ok(())
+            }
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Serialization,
+            }
+            .into()),
+        }
+    }
+    pub fn insert_model(&mut self, key: &T::Key, value: Model<T::Value>) -> Result<(), Error> {
+        use patch_db::ModelExt;
+        use serde::ser::Error;
+        let v = value.into_value();
+        match &mut self.value {
+            Value::Object(o) => {
+                o.insert(InternedString::intern(key.as_ref()), v);
+                Ok(())
+            }
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Serialization,
+            }
+            .into()),
+        }
+    }
+}
+
+impl<T: Map> Model<T>
+where
+    T::Key: DeserializeOwned + Ord + Clone,
+{
+    pub fn keys(&self) -> Result<Vec<T::Key>, Error> {
+        use serde::de::Error;
+        use serde::Deserialize;
+        match &self.value {
+            Value::Object(o) => o
+                .keys()
+                .cloned()
+                .map(|k| {
+                    T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(k))
+                        .map_err(|e| {
+                            patch_db::value::Error {
+                                kind: patch_db::value::ErrorKind::Deserialization,
+                                source: e,
+                            }
+                            .into()
+                        })
+                })
+                .collect(),
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Deserialization,
+            }
+            .into()),
+        }
+    }
+
+    pub fn into_entries(self) -> Result<Vec<(T::Key, Model<T::Value>)>, Error> {
+        use patch_db::ModelExt;
+        use serde::de::Error;
+        use serde::Deserialize;
+        match self.value {
+            Value::Object(o) => o
+                .into_iter()
+                .map(|(k, v)| {
+                    Ok((
+                        T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
+                            k,
+                        ))
+                        .with_kind(ErrorKind::Deserialization)?,
+                        Model::from_value(v),
+                    ))
+                })
+                .collect(),
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Deserialization,
+            }
+            .into()),
+        }
+    }
+    pub fn as_entries(&self) -> Result<Vec<(T::Key, &Model<T::Value>)>, Error> {
+        use patch_db::ModelExt;
+        use serde::de::Error;
+        use serde::Deserialize;
+        match &self.value {
+            Value::Object(o) => o
+                .iter()
+                .map(|(k, v)| {
+                    Ok((
+                        T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
+                            k.clone(),
+                        ))
+                        .with_kind(ErrorKind::Deserialization)?,
+                        Model::value_as(v),
+                    ))
+                })
+                .collect(),
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Deserialization,
+            }
+            .into()),
+        }
+    }
+    pub fn as_entries_mut(&mut self) -> Result<Vec<(T::Key, &mut Model<T::Value>)>, Error> {
+        use patch_db::ModelExt;
+        use serde::de::Error;
+        use serde::Deserialize;
+        match &mut self.value {
+            Value::Object(o) => o
+                .iter_mut()
+                .map(|(k, v)| {
+                    Ok((
+                        T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
+                            k.clone(),
+                        ))
+                        .with_kind(ErrorKind::Deserialization)?,
+                        Model::value_as_mut(v),
+                    ))
+                })
+                .collect(),
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Deserialization,
+            }
+            .into()),
+        }
+    }
+}
+impl<T: Map> Model<T>
+where
+    T::Key: AsRef<str>,
+{
+    pub fn into_idx(self, key: &T::Key) -> Option<Model<T::Value>> {
+        use patch_db::ModelExt;
+        match &self.value {
+            Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute(|v| {
+                use patch_db::value::index::Index;
+                key.as_ref().index_into_owned(v).unwrap()
+            })),
+            _ => None,
+        }
+    }
+    pub fn as_idx<'a>(&'a self, key: &T::Key) -> Option<&'a Model<T::Value>> {
+        use patch_db::ModelExt;
+        match &self.value {
+            Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute_ref(|v| {
+                use patch_db::value::index::Index;
+                key.as_ref().index_into(v).unwrap()
+            })),
+            _ => None,
+        }
+    }
+    pub fn as_idx_mut<'a>(&'a mut self, key: &T::Key) -> Option<&'a mut Model<T::Value>> {
+        use patch_db::ModelExt;
+        match &mut self.value {
+            Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute_mut(|v| {
+                use patch_db::value::index::Index;
+                key.as_ref().index_or_insert(v)
+            })),
+            _ => None,
+        }
+    }
+    pub fn remove(&mut self, key: &T::Key) -> Result<Option<Model<T::Value>>, Error> {
+        use serde::ser::Error;
+        match &mut self.value {
+            Value::Object(o) => {
+                let v = o.remove(key.as_ref());
+                Ok(v.map(patch_db::ModelExt::from_value))
+            }
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Serialization,
+            }
+            .into()),
+        }
+    }
+}

core/startos/src/dependencies.rs

@@ -0,0 +1,363 @@
+use std::collections::BTreeMap;
+use std::time::Duration;
+
+use color_eyre::eyre::eyre;
+use emver::VersionRange;
+use models::OptionExt;
+use rand::SeedableRng;
+use rpc_toolkit::command;
+use serde::{Deserialize, Serialize};
+use tracing::instrument;
+
+use crate::config::action::ConfigRes;
+use crate::config::spec::PackagePointerSpec;
+use crate::config::{not_found, Config, ConfigSpec, ConfigureContext};
+use crate::context::RpcContext;
+use crate::db::model::{CurrentDependencies, Database};
+use crate::prelude::*;
+use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
+use crate::s9pk::manifest::{Manifest, PackageId};
+use crate::status::DependencyConfigErrors;
+use crate::util::serde::display_serializable;
+use crate::util::{display_none, Version};
+use crate::volume::Volumes;
+use crate::Error;
+
+#[command(subcommands(configure))]
+pub fn dependency() -> Result<(), Error> {
+    Ok(())
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
+#[model = "Model<Self>"]
+pub struct Dependencies(pub BTreeMap<PackageId, DepInfo>);
+impl Map for Dependencies {
+    type Key = PackageId;
+    type Value = DepInfo;
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+#[serde(tag = "type")]
+pub enum DependencyRequirement {
+    OptIn { how: String },
+    OptOut { how: String },
+    Required,
+}
+impl DependencyRequirement {
+    pub fn required(&self) -> bool {
+        matches!(self, &DependencyRequirement::Required)
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct DepInfo {
+    pub version: VersionRange,
+    pub requirement: DependencyRequirement,
+    pub description: Option<String>,
+    #[serde(default)]
+    pub config: Option<DependencyConfig>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct DependencyConfig {
+    check: PackageProcedure,
+    auto_configure: PackageProcedure,
+}
+impl DependencyConfig {
+    pub async fn check(
+        &self,
+        ctx: &RpcContext,
+        dependent_id: &PackageId,
+        dependent_version: &Version,
+        dependent_volumes: &Volumes,
+        dependency_id: &PackageId,
+        dependency_config: &Config,
+    ) -> Result<Result<NoOutput, String>, Error> {
+        Ok(self
+            .check
+            .sandboxed(
+                ctx,
+                dependent_id,
+                dependent_version,
+                dependent_volumes,
+                Some(dependency_config),
+                None,
+                ProcedureName::Check(dependency_id.clone()),
+            )
+            .await?
+            .map_err(|(_, e)| e))
+    }
+    pub async fn auto_configure(
+        &self,
+        ctx: &RpcContext,
+        dependent_id: &PackageId,
+        dependent_version: &Version,
+        dependent_volumes: &Volumes,
+        old: &Config,
+    ) -> Result<Config, Error> {
+        self.auto_configure
+            .sandboxed(
+                ctx,
+                dependent_id,
+                dependent_version,
+                dependent_volumes,
+                Some(old),
+                None,
+                ProcedureName::AutoConfig(dependent_id.clone()),
+            )
+            .await?
+            .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))
+    }
+}
+
+#[command(
+    subcommands(self(configure_impl(async)), configure_dry),
+    display(display_none)
+)]
+pub async fn configure(
+    #[arg(rename = "dependent-id")] dependent_id: PackageId,
+    #[arg(rename = "dependency-id")] dependency_id: PackageId,
+) -> Result<(PackageId, PackageId), Error> {
+    Ok((dependent_id, dependency_id))
+}
+
+pub async fn configure_impl(
+    ctx: RpcContext,
+    (pkg_id, dep_id): (PackageId, PackageId),
+) -> Result<(), Error> {
+    let breakages = BTreeMap::new();
+    let overrides = Default::default();
+    let ConfigDryRes {
+        old_config: _,
+        new_config,
+        spec: _,
+    } = configure_logic(ctx.clone(), (pkg_id, dep_id.clone())).await?;
+
+    let configure_context = ConfigureContext {
+        breakages,
+        timeout: Some(Duration::from_secs(3).into()),
+        config: Some(new_config),
+        dry_run: false,
+        overrides,
+    };
+    crate::config::configure(&ctx, &dep_id, configure_context).await?;
+    Ok(())
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct ConfigDryRes {
+    pub old_config: Config,
+    pub new_config: Config,
+    pub spec: ConfigSpec,
+}
+
+#[command(rename = "dry", display(display_serializable))]
+#[instrument(skip_all)]
+pub async fn configure_dry(
+    #[context] ctx: RpcContext,
+    #[parent_data] (pkg_id, dependency_id): (PackageId, PackageId),
+) -> Result<ConfigDryRes, Error> {
+    configure_logic(ctx, (pkg_id, dependency_id)).await
+}
+
+pub async fn configure_logic(
+    ctx: RpcContext,
+    (pkg_id, dependency_id): (PackageId, PackageId),
+) -> Result<ConfigDryRes, Error> {
+    let db = ctx.db.peek().await;
+    let pkg = db
+        .as_package_data()
+        .as_idx(&pkg_id)
+        .or_not_found(&pkg_id)?
+        .as_installed()
+        .or_not_found(&pkg_id)?;
+    let pkg_version = pkg.as_manifest().as_version().de()?;
+    let pkg_volumes = pkg.as_manifest().as_volumes().de()?;
+    let dependency = db
+        .as_package_data()
+        .as_idx(&dependency_id)
+        .or_not_found(&dependency_id)?
+        .as_installed()
+        .or_not_found(&dependency_id)?;
+    let dependency_config_action = dependency
+        .as_manifest()
+        .as_config()
+        .de()?
+        .ok_or_else(|| not_found!("Manifest Config"))?;
+    let dependency_version = dependency.as_manifest().as_version().de()?;
+    let dependency_volumes = dependency.as_manifest().as_volumes().de()?;
+    let dependency = pkg
+        .as_manifest()
+        .as_dependencies()
+        .as_idx(&dependency_id)
+        .or_not_found(&dependency_id)?;
+
+    let ConfigRes {
+        config: maybe_config,
+        spec,
+    } = dependency_config_action
+        .get(
+            &ctx,
+            &dependency_id,
+            &dependency_version,
+            &dependency_volumes,
+        )
+        .await?;
+
+    let old_config = if let Some(config) = maybe_config {
+        config
+    } else {
+        spec.gen(
+            &mut rand::rngs::StdRng::from_entropy(),
+            &Some(Duration::new(10, 0)),
+        )?
+    };
+
+    let new_config = dependency
+        .as_config()
+        .de()?
+        .ok_or_else(|| not_found!("Config"))?
+        .auto_configure
+        .sandboxed(
+            &ctx,
+            &pkg_id,
+            &pkg_version,
+            &pkg_volumes,
+            Some(&old_config),
+            None,
+            ProcedureName::AutoConfig(dependency_id.clone()),
+        )
+        .await?
+        .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))?;
+
+    Ok(ConfigDryRes {
+        old_config,
+        new_config,
+        spec,
+    })
+}
+
+#[instrument(skip_all)]
+pub fn add_dependent_to_current_dependents_lists(
+    db: &mut Model<Database>,
+    dependent_id: &PackageId,
+    current_dependencies: &CurrentDependencies,
+) -> Result<(), Error> {
+    for (dependency, dep_info) in &current_dependencies.0 {
+        if let Some(dependency_dependents) = db
+            .as_package_data_mut()
+            .as_idx_mut(dependency)
+            .and_then(|pde| pde.as_installed_mut())
+            .map(|i| i.as_current_dependents_mut())
+        {
+            dependency_dependents.insert(dependent_id, dep_info)?;
+        }
+    }
+    Ok(())
+}
+
+pub fn set_dependents_with_live_pointers_to_needs_config(
+    db: &mut Peeked,
+    id: &PackageId,
+) -> Result<Vec<(PackageId, Version)>, Error> {
+    let mut res = Vec::new();
+    for (dep, info) in db
+        .as_package_data()
+        .as_idx(id)
+        .or_not_found(id)?
+        .as_installed()
+        .or_not_found(id)?
+        .as_current_dependents()
+        .de()?
+        .0
+    {
+        if info.pointers.iter().any(|ptr| match ptr {
+            // dependency id matches the package being uninstalled
+            PackagePointerSpec::TorAddress(ptr) => &ptr.package_id == id && &dep != id,
+            PackagePointerSpec::LanAddress(ptr) => &ptr.package_id == id && &dep != id,
+            // we never need to retarget these
+            PackagePointerSpec::TorKey(_) => false,
+            PackagePointerSpec::Config(_) => false,
+        }) {
+            let installed = db
+                .as_package_data_mut()
+                .as_idx_mut(&dep)
+                .or_not_found(&dep)?
+                .as_installed_mut()
+                .or_not_found(&dep)?;
+            let version = installed.as_manifest().as_version().de()?;
+            let configured = installed.as_status_mut().as_configured_mut();
+            if configured.de()? {
+                configured.ser(&false)?;
+                res.push((dep, version));
+            }
+        }
+    }
+    Ok(res)
+}
+
+#[instrument(skip_all)]
+pub async fn compute_dependency_config_errs(
+    ctx: &RpcContext,
+    db: &Peeked,
+    manifest: &Manifest,
+    current_dependencies: &CurrentDependencies,
+    dependency_config: &BTreeMap<PackageId, Config>,
+) -> Result<DependencyConfigErrors, Error> {
+    let mut dependency_config_errs = BTreeMap::new();
+    for (dependency, _dep_info) in current_dependencies
+        .0
+        .iter()
+        .filter(|(dep_id, _)| dep_id != &&manifest.id)
+    {
+        // check if config passes dependency check
+        if let Some(cfg) = &manifest
+            .dependencies
+            .0
+            .get(dependency)
+            .or_not_found(dependency)?
+            .config
+        {
+            if let Err(error) = cfg
+                .check(
+                    ctx,
+                    &manifest.id,
+                    &manifest.version,
+                    &manifest.volumes,
+                    dependency,
+                    &if let Some(config) = dependency_config.get(dependency) {
+                        config.clone()
+                    } else if let Some(manifest) = db
+                        .as_package_data()
+                        .as_idx(dependency)
+                        .and_then(|pde| pde.as_installed())
+                        .map(|i| i.as_manifest().de())
+                        .transpose()?
+                    {
+                        if let Some(config) = &manifest.config {
+                            config
+                                .get(ctx, &manifest.id, &manifest.version, &manifest.volumes)
+                                .await?
+                                .config
+                                .unwrap_or_default()
+                        } else {
+                            Config::default()
+                        }
+                    } else {
+                        Config::default()
+                    },
+                )
+                .await?
+            {
+                dependency_config_errs.insert(dependency.clone(), error);
+            }
+        }
+    }
+    Ok(DependencyConfigErrors(dependency_config_errs))
+}

core/startos/src/developer/mod.rs

@@ -0,0 +1,55 @@
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+use ed25519::pkcs8::EncodePrivateKey;
+use ed25519::PublicKeyBytes;
+use ed25519_dalek::{SigningKey, VerifyingKey};
+use rpc_toolkit::command;
+use tracing::instrument;
+
+use crate::context::SdkContext;
+use crate::util::display_none;
+use crate::{Error, ResultExt};
+
+#[command(cli_only, blocking, display(display_none))]
+#[instrument(skip_all)]
+pub fn init(#[context] ctx: SdkContext) -> Result<(), Error> {
+    if !ctx.developer_key_path.exists() {
+        let parent = ctx.developer_key_path.parent().unwrap_or(Path::new("/"));
+        if !parent.exists() {
+            std::fs::create_dir_all(parent)
+                .with_ctx(|_| (crate::ErrorKind::Filesystem, parent.display().to_string()))?;
+        }
+        tracing::info!("Generating new developer key...");
+        let secret = SigningKey::generate(&mut rand::thread_rng());
+        tracing::info!("Writing key to {}", ctx.developer_key_path.display());
+        let keypair_bytes = ed25519::KeypairBytes {
+            secret_key: secret.to_bytes(),
+            public_key: Some(PublicKeyBytes(VerifyingKey::from(&secret).to_bytes())),
+        };
+        let mut dev_key_file = File::create(&ctx.developer_key_path)?;
+        dev_key_file.write_all(
+            keypair_bytes
+                .to_pkcs8_pem(base64ct::LineEnding::default())
+                .with_kind(crate::ErrorKind::Pem)?
+                .as_bytes(),
+        )?;
+        dev_key_file.sync_all()?;
+        println!(
+            "New developer key generated at {}",
+            ctx.developer_key_path.display()
+        );
+    } else {
+        println!(
+            "Developer key already exists at {}",
+            ctx.developer_key_path.display()
+        );
+    }
+    Ok(())
+}
+
+#[command(subcommands(crate::s9pk::verify, crate::config::verify_spec))]
+pub fn verify() -> Result<(), Error> {
+    Ok(())
+}

core/startos/src/diagnostic.rs

@@ -0,0 +1,72 @@
+use std::path::Path;
+use std::sync::Arc;
+
+use rpc_toolkit::command;
+use rpc_toolkit::yajrc::RpcError;
+
+use crate::context::DiagnosticContext;
+use crate::disk::repair;
+use crate::init::SYSTEM_REBUILD_PATH;
+use crate::logs::{fetch_logs, LogResponse, LogSource};
+use crate::shutdown::Shutdown;
+use crate::util::display_none;
+use crate::Error;
+
+#[command(subcommands(error, logs, exit, restart, forget_disk, disk, rebuild))]
+pub fn diagnostic() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command]
+pub fn error(#[context] ctx: DiagnosticContext) -> Result<Arc<RpcError>, Error> {
+    Ok(ctx.error.clone())
+}
+
+#[command(rpc_only)]
+pub async fn logs(
+    #[arg] limit: Option<usize>,
+    #[arg] cursor: Option<String>,
+    #[arg] before: bool,
+) -> Result<LogResponse, Error> {
+    Ok(fetch_logs(LogSource::System, limit, cursor, before).await?)
+}
+
+#[command(display(display_none))]
+pub fn exit(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
+    ctx.shutdown.send(None).expect("receiver dropped");
+    Ok(())
+}
+
+#[command(display(display_none))]
+pub fn restart(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
+    ctx.shutdown
+        .send(Some(Shutdown {
+            export_args: ctx
+                .disk_guid
+                .clone()
+                .map(|guid| (guid, ctx.datadir.clone())),
+            restart: true,
+        }))
+        .expect("receiver dropped");
+    Ok(())
+}
+
+#[command(display(display_none))]
+pub async fn rebuild(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
+    tokio::fs::write(SYSTEM_REBUILD_PATH, b"").await?;
+    restart(ctx)
+}
+
+#[command(subcommands(forget_disk, repair))]
+pub fn disk() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(rename = "forget", display(display_none))]
+pub async fn forget_disk() -> Result<(), Error> {
+    let disk_guid = Path::new("/media/embassy/config/disk.guid");
+    if tokio::fs::metadata(disk_guid).await.is_ok() {
+        tokio::fs::remove_file(disk_guid).await?;
+    }
+    Ok(())
+}

core/startos/src/disk/fsck/btrfs.rs

@@ -0,0 +1,32 @@
+use std::path::Path;
+
+use tokio::process::Command;
+use tracing::instrument;
+
+use crate::disk::fsck::RequiresReboot;
+use crate::util::Invoke;
+use crate::Error;
+
+#[instrument(skip_all)]
+pub async fn btrfs_check_readonly(logicalname: impl AsRef<Path>) -> Result<RequiresReboot, Error> {
+    Command::new("btrfs")
+        .arg("check")
+        .arg("--readonly")
+        .arg(logicalname.as_ref())
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+
+    Ok(RequiresReboot(false))
+}
+
+pub async fn btrfs_check_repair(logicalname: impl AsRef<Path>) -> Result<RequiresReboot, Error> {
+    Command::new("btrfs")
+        .arg("check")
+        .arg("--repair")
+        .arg("--force")
+        .arg(logicalname.as_ref())
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+
+    Ok(RequiresReboot(false))
+}

core/startos/src/disk/fsck/ext4.rs

@@ -0,0 +1,95 @@
+use std::ffi::OsStr;
+use std::path::Path;
+
+use color_eyre::eyre::eyre;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use tokio::process::Command;
+use tracing::instrument;
+
+use crate::disk::fsck::RequiresReboot;
+use crate::Error;
+
+#[instrument(skip_all)]
+pub async fn e2fsck_preen(
+    logicalname: impl AsRef<Path> + std::fmt::Debug,
+) -> Result<RequiresReboot, Error> {
+    e2fsck_runner(Command::new("e2fsck").arg("-p"), logicalname).await
+}
+
+fn backup_existing_undo_file<'a>(path: &'a Path) -> BoxFuture<'a, Result<(), Error>> {
+    async move {
+        if tokio::fs::metadata(path).await.is_ok() {
+            let bak = path.with_extension(format!(
+                "{}.bak",
+                path.extension()
+                    .and_then(|s| s.to_str())
+                    .unwrap_or_default()
+            ));
+            backup_existing_undo_file(&bak).await?;
+            tokio::fs::rename(path, &bak).await?;
+        }
+        Ok(())
+    }
+    .boxed()
+}
+
+#[instrument(skip_all)]
+pub async fn e2fsck_aggressive(
+    logicalname: impl AsRef<Path> + std::fmt::Debug,
+) -> Result<RequiresReboot, Error> {
+    let undo_path = Path::new("/media/embassy/config")
+        .join(
+            logicalname
+                .as_ref()
+                .file_name()
+                .unwrap_or(OsStr::new("unknown")),
+        )
+        .with_extension("e2undo");
+    backup_existing_undo_file(&undo_path).await?;
+    e2fsck_runner(
+        Command::new("e2fsck").arg("-y").arg("-z").arg(undo_path),
+        logicalname,
+    )
+    .await
+}
+
+async fn e2fsck_runner(
+    e2fsck_cmd: &mut Command,
+    logicalname: impl AsRef<Path> + std::fmt::Debug,
+) -> Result<RequiresReboot, Error> {
+    let e2fsck_out = e2fsck_cmd.arg(logicalname.as_ref()).output().await?;
+    let e2fsck_stderr = String::from_utf8(e2fsck_out.stderr)?;
+    let code = e2fsck_out.status.code().ok_or_else(|| {
+        Error::new(
+            eyre!("e2fsck: process terminated by signal"),
+            crate::ErrorKind::DiskManagement,
+        )
+    })?;
+    if code & 4 != 0 {
+        tracing::error!(
+            "some filesystem errors NOT corrected on {}:\n{}",
+            logicalname.as_ref().display(),
+            e2fsck_stderr,
+        );
+    } else if code & 1 != 0 {
+        tracing::warn!(
+            "filesystem errors corrected on {}:\n{}",
+            logicalname.as_ref().display(),
+            e2fsck_stderr,
+        );
+    }
+    if code < 8 {
+        if code & 2 != 0 {
+            tracing::warn!("reboot required");
+            Ok(RequiresReboot(true))
+        } else {
+            Ok(RequiresReboot(false))
+        }
+    } else {
+        Err(Error::new(
+            eyre!("e2fsck: {}", e2fsck_stderr),
+            crate::ErrorKind::DiskManagement,
+        ))
+    }
+}

core/startos/src/disk/fsck/mod.rs

@@ -0,0 +1,70 @@
+use std::path::Path;
+
+use color_eyre::eyre::eyre;
+use tokio::process::Command;
+
+use crate::disk::fsck::btrfs::{btrfs_check_readonly, btrfs_check_repair};
+use crate::disk::fsck::ext4::{e2fsck_aggressive, e2fsck_preen};
+use crate::util::Invoke;
+use crate::Error;
+
+pub mod btrfs;
+pub mod ext4;
+
+#[derive(Debug, Clone, Copy)]
+#[must_use]
+pub struct RequiresReboot(pub bool);
+impl std::ops::BitOrAssign for RequiresReboot {
+    fn bitor_assign(&mut self, rhs: Self) {
+        self.0 |= rhs.0
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum RepairStrategy {
+    Preen,
+    Aggressive,
+}
+impl RepairStrategy {
+    pub async fn fsck(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match &*String::from_utf8(
+            Command::new("grub-probe")
+                .arg("-d")
+                .arg(logicalname.as_ref())
+                .invoke(crate::ErrorKind::DiskManagement)
+                .await?,
+        )?
+        .trim()
+        {
+            "ext2" => self.e2fsck(logicalname).await,
+            "btrfs" => self.btrfs_check(logicalname).await,
+            fs => {
+                return Err(Error::new(
+                    eyre!("Unknown filesystem {fs}"),
+                    crate::ErrorKind::DiskManagement,
+                ))
+            }
+        }
+    }
+    pub async fn e2fsck(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match self {
+            RepairStrategy::Preen => e2fsck_preen(logicalname).await,
+            RepairStrategy::Aggressive => e2fsck_aggressive(logicalname).await,
+        }
+    }
+    pub async fn btrfs_check(
+        &self,
+        logicalname: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<RequiresReboot, Error> {
+        match self {
+            RepairStrategy::Preen => btrfs_check_readonly(logicalname).await,
+            RepairStrategy::Aggressive => btrfs_check_repair(logicalname).await,
+        }
+    }
+}

core/startos/src/disk/main.rs

@@ -0,0 +1,337 @@
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+use color_eyre::eyre::eyre;
+use tokio::process::Command;
+use tracing::instrument;
+
+use super::fsck::{RepairStrategy, RequiresReboot};
+use super::util::pvscan;
+use crate::disk::mount::filesystem::block_dev::mount;
+use crate::disk::mount::filesystem::ReadWrite;
+use crate::disk::mount::util::unmount;
+use crate::util::Invoke;
+use crate::{Error, ErrorKind, ResultExt};
+
+pub const PASSWORD_PATH: &'static str = "/run/embassy/password";
+pub const DEFAULT_PASSWORD: &'static str = "password";
+pub const MAIN_FS_SIZE: FsSize = FsSize::Gigabytes(8);
+
+#[instrument(skip_all)]
+pub async fn create<I, P>(
+    disks: &I,
+    pvscan: &BTreeMap<PathBuf, Option<String>>,
+    datadir: impl AsRef<Path>,
+    password: Option<&str>,
+) -> Result<String, Error>
+where
+    for<'a> &'a I: IntoIterator<Item = &'a P>,
+    P: AsRef<Path>,
+{
+    let guid = create_pool(disks, pvscan, password.is_some()).await?;
+    create_all_fs(&guid, &datadir, password).await?;
+    export(&guid, datadir).await?;
+    Ok(guid)
+}
+
+#[instrument(skip_all)]
+pub async fn create_pool<I, P>(
+    disks: &I,
+    pvscan: &BTreeMap<PathBuf, Option<String>>,
+    encrypted: bool,
+) -> Result<String, Error>
+where
+    for<'a> &'a I: IntoIterator<Item = &'a P>,
+    P: AsRef<Path>,
+{
+    Command::new("dmsetup")
+        .arg("remove_all") // TODO: find a higher finesse way to do this for portability reasons
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    for disk in disks {
+        if pvscan.contains_key(disk.as_ref()) {
+            Command::new("pvremove")
+                .arg("-yff")
+                .arg(disk.as_ref())
+                .invoke(crate::ErrorKind::DiskManagement)
+                .await?;
+        }
+        tokio::fs::write(disk.as_ref(), &[0; 2048]).await?; // wipe partition table
+        Command::new("pvcreate")
+            .arg("-yff")
+            .arg(disk.as_ref())
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+    }
+    let mut guid = format!(
+        "EMBASSY_{}",
+        base32::encode(
+            base32::Alphabet::RFC4648 { padding: false },
+            &rand::random::<[u8; 32]>(),
+        )
+    );
+    if !encrypted {
+        guid += "_UNENC";
+    }
+    let mut cmd = Command::new("vgcreate");
+    cmd.arg("-y").arg(&guid);
+    for disk in disks {
+        cmd.arg(disk.as_ref());
+    }
+    cmd.invoke(crate::ErrorKind::DiskManagement).await?;
+    Ok(guid)
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum FsSize {
+    Gigabytes(usize),
+    FreePercentage(usize),
+}
+
+#[instrument(skip_all)]
+pub async fn create_fs<P: AsRef<Path>>(
+    guid: &str,
+    datadir: P,
+    name: &str,
+    size: FsSize,
+    password: Option<&str>,
+) -> Result<(), Error> {
+    let mut cmd = Command::new("lvcreate");
+    match size {
+        FsSize::Gigabytes(a) => cmd.arg("-L").arg(format!("{}G", a)),
+        FsSize::FreePercentage(a) => cmd.arg("-l").arg(format!("{}%FREE", a)),
+    };
+    cmd.arg("-y")
+        .arg("-n")
+        .arg(name)
+        .arg(guid)
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    let mut blockdev_path = Path::new("/dev").join(guid).join(name);
+    if let Some(password) = password {
+        if let Some(parent) = Path::new(PASSWORD_PATH).parent() {
+            tokio::fs::create_dir_all(parent).await?;
+        }
+        tokio::fs::write(PASSWORD_PATH, password)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksFormat")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksOpen")
+            .arg("--allow-discards")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .arg(format!("{}_{}", guid, name))
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::remove_file(PASSWORD_PATH)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        blockdev_path = Path::new("/dev/mapper").join(format!("{}_{}", guid, name));
+    }
+    Command::new("mkfs.btrfs")
+        .arg(&blockdev_path)
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    mount(&blockdev_path, datadir.as_ref().join(name), ReadWrite).await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn create_all_fs<P: AsRef<Path>>(
+    guid: &str,
+    datadir: P,
+    password: Option<&str>,
+) -> Result<(), Error> {
+    create_fs(guid, &datadir, "main", MAIN_FS_SIZE, password).await?;
+    create_fs(
+        guid,
+        &datadir,
+        "package-data",
+        FsSize::FreePercentage(100),
+        password,
+    )
+    .await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn unmount_fs<P: AsRef<Path>>(guid: &str, datadir: P, name: &str) -> Result<(), Error> {
+    unmount(datadir.as_ref().join(name)).await?;
+    if !guid.ends_with("_UNENC") {
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksClose")
+            .arg(format!("{}_{}", guid, name))
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+    }
+
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn unmount_all_fs<P: AsRef<Path>>(guid: &str, datadir: P) -> Result<(), Error> {
+    unmount_fs(guid, &datadir, "main").await?;
+    unmount_fs(guid, &datadir, "package-data").await?;
+    Command::new("dmsetup")
+        .arg("remove_all") // TODO: find a higher finesse way to do this for portability reasons
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn export<P: AsRef<Path>>(guid: &str, datadir: P) -> Result<(), Error> {
+    Command::new("sync").invoke(ErrorKind::Filesystem).await?;
+    unmount_all_fs(guid, datadir).await?;
+    Command::new("vgchange")
+        .arg("-an")
+        .arg(guid)
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    Command::new("vgexport")
+        .arg(guid)
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn import<P: AsRef<Path>>(
+    guid: &str,
+    datadir: P,
+    repair: RepairStrategy,
+    password: Option<&str>,
+) -> Result<RequiresReboot, Error> {
+    let scan = pvscan().await?;
+    if scan
+        .values()
+        .filter_map(|a| a.as_ref())
+        .filter(|a| a.starts_with("EMBASSY_"))
+        .next()
+        .is_none()
+    {
+        return Err(Error::new(
+            eyre!("StartOS disk not found."),
+            crate::ErrorKind::DiskNotAvailable,
+        ));
+    }
+    if !scan
+        .values()
+        .filter_map(|a| a.as_ref())
+        .any(|id| id == guid)
+    {
+        return Err(Error::new(
+            eyre!("A StartOS disk was found, but it is not the correct disk for this device."),
+            crate::ErrorKind::IncorrectDisk,
+        ));
+    }
+    Command::new("dmsetup")
+        .arg("remove_all") // TODO: find a higher finesse way to do this for portability reasons
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    match Command::new("vgimport")
+        .arg(guid)
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await
+    {
+        Ok(_) => Ok(()),
+        Err(e)
+            if format!("{}", e.source)
+                .lines()
+                .any(|l| l.trim() == format!("Volume group \"{}\" is not exported", guid)) =>
+        {
+            Ok(())
+        }
+        Err(e) => Err(e),
+    }?;
+    Command::new("vgchange")
+        .arg("-ay")
+        .arg(guid)
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    mount_all_fs(guid, datadir, repair, password).await
+}
+
+#[instrument(skip_all)]
+pub async fn mount_fs<P: AsRef<Path>>(
+    guid: &str,
+    datadir: P,
+    name: &str,
+    repair: RepairStrategy,
+    password: Option<&str>,
+) -> Result<RequiresReboot, Error> {
+    let orig_path = Path::new("/dev").join(guid).join(name);
+    let mut blockdev_path = orig_path.clone();
+    let full_name = format!("{}_{}", guid, name);
+    if !guid.ends_with("_UNENC") {
+        let password = password.unwrap_or(DEFAULT_PASSWORD);
+        if let Some(parent) = Path::new(PASSWORD_PATH).parent() {
+            tokio::fs::create_dir_all(parent).await?;
+        }
+        tokio::fs::write(PASSWORD_PATH, password)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksOpen")
+            .arg(format!("--key-file={}", PASSWORD_PATH))
+            .arg(format!("--keyfile-size={}", password.len()))
+            .arg(&blockdev_path)
+            .arg(&full_name)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::remove_file(PASSWORD_PATH)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?;
+        blockdev_path = Path::new("/dev/mapper").join(&full_name);
+    }
+    let reboot = repair.fsck(&blockdev_path).await?;
+
+    if !guid.ends_with("_UNENC") {
+        // Backup LUKS header if e2fsck succeeded
+        let luks_folder = Path::new("/media/embassy/config/luks");
+        tokio::fs::create_dir_all(luks_folder).await?;
+        let tmp_luks_bak = luks_folder.join(format!(".{full_name}.luks.bak.tmp"));
+        if tokio::fs::metadata(&tmp_luks_bak).await.is_ok() {
+            tokio::fs::remove_file(&tmp_luks_bak).await?;
+        }
+        let luks_bak = luks_folder.join(format!("{full_name}.luks.bak"));
+        Command::new("cryptsetup")
+            .arg("-q")
+            .arg("luksHeaderBackup")
+            .arg("--header-backup-file")
+            .arg(&tmp_luks_bak)
+            .arg(&orig_path)
+            .invoke(crate::ErrorKind::DiskManagement)
+            .await?;
+        tokio::fs::rename(&tmp_luks_bak, &luks_bak).await?;
+    }
+
+    mount(&blockdev_path, datadir.as_ref().join(name), ReadWrite).await?;
+
+    Ok(reboot)
+}
+
+#[instrument(skip_all)]
+pub async fn mount_all_fs<P: AsRef<Path>>(
+    guid: &str,
+    datadir: P,
+    repair: RepairStrategy,
+    password: Option<&str>,
+) -> Result<RequiresReboot, Error> {
+    let mut reboot = RequiresReboot(false);
+    reboot |= mount_fs(guid, &datadir, "main", repair, password).await?;
+    reboot |= mount_fs(guid, &datadir, "package-data", repair, password).await?;
+    Ok(reboot)
+}

core/startos/src/disk/mod.rs

@@ -0,0 +1,118 @@
+use std::path::{Path, PathBuf};
+
+use clap::ArgMatches;
+use rpc_toolkit::command;
+use serde::{Deserialize, Serialize};
+
+use crate::context::RpcContext;
+use crate::disk::util::DiskInfo;
+use crate::util::display_none;
+use crate::util::serde::{display_serializable, IoFormat};
+use crate::Error;
+
+pub mod fsck;
+pub mod main;
+pub mod mount;
+pub mod util;
+
+pub const BOOT_RW_PATH: &str = "/media/boot-rw";
+pub const REPAIR_DISK_PATH: &str = "/media/embassy/config/repair-disk";
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct OsPartitionInfo {
+    pub efi: Option<PathBuf>,
+    pub bios: Option<PathBuf>,
+    pub boot: PathBuf,
+    pub root: PathBuf,
+}
+impl OsPartitionInfo {
+    pub fn contains(&self, logicalname: impl AsRef<Path>) -> bool {
+        self.efi
+            .as_ref()
+            .map(|p| p == logicalname.as_ref())
+            .unwrap_or(false)
+            || self
+                .bios
+                .as_ref()
+                .map(|p| p == logicalname.as_ref())
+                .unwrap_or(false)
+            || &*self.boot == logicalname.as_ref()
+            || &*self.root == logicalname.as_ref()
+    }
+}
+
+#[command(subcommands(list, repair))]
+pub fn disk() -> Result<(), Error> {
+    Ok(())
+}
+
+fn display_disk_info(info: Vec<DiskInfo>, matches: &ArgMatches) {
+    use prettytable::*;
+
+    if matches.is_present("format") {
+        return display_serializable(info, matches);
+    }
+
+    let mut table = Table::new();
+    table.add_row(row![bc =>
+        "LOGICALNAME",
+        "LABEL",
+        "CAPACITY",
+        "USED",
+        "EMBASSY OS VERSION"
+    ]);
+    for disk in info {
+        let row = row![
+            disk.logicalname.display(),
+            "N/A",
+            &format!("{:.2} GiB", disk.capacity as f64 / 1024.0 / 1024.0 / 1024.0),
+            "N/A",
+            "N/A",
+        ];
+        table.add_row(row);
+        for part in disk.partitions {
+            let row = row![
+                part.logicalname.display(),
+                if let Some(label) = part.label.as_ref() {
+                    label
+                } else {
+                    "N/A"
+                },
+                part.capacity,
+                if let Some(used) = part
+                    .used
+                    .map(|u| format!("{:.2} GiB", u as f64 / 1024.0 / 1024.0 / 1024.0))
+                    .as_ref()
+                {
+                    used
+                } else {
+                    "N/A"
+                },
+                if let Some(eos) = part.embassy_os.as_ref() {
+                    eos.version.as_str()
+                } else {
+                    "N/A"
+                },
+            ];
+            table.add_row(row);
+        }
+    }
+    table.print_tty(false).unwrap();
+}
+
+#[command(display(display_disk_info))]
+pub async fn list(
+    #[context] ctx: RpcContext,
+    #[allow(unused_variables)]
+    #[arg]
+    format: Option<IoFormat>,
+) -> Result<Vec<DiskInfo>, Error> {
+    crate::disk::util::list(&ctx.os_partitions).await
+}
+
+#[command(display(display_none))]
+pub async fn repair() -> Result<(), Error> {
+    tokio::fs::write(REPAIR_DISK_PATH, b"").await?;
+    Ok(())
+}

core/startos/src/disk/mount/backup.rs

@@ -0,0 +1,262 @@
+use std::path::{Path, PathBuf};
+
+use color_eyre::eyre::eyre;
+use helpers::AtomicFile;
+use tokio::io::AsyncWriteExt;
+use tracing::instrument;
+
+use super::filesystem::ecryptfs::EcryptFS;
+use super::guard::{GenericMountGuard, TmpMountGuard};
+use super::util::{bind, unmount};
+use crate::auth::check_password;
+use crate::backup::target::BackupInfo;
+use crate::disk::mount::filesystem::ReadWrite;
+use crate::disk::util::EmbassyOsRecoveryInfo;
+use crate::middleware::encrypt::{decrypt_slice, encrypt_slice};
+use crate::s9pk::manifest::PackageId;
+use crate::util::serde::IoFormat;
+use crate::util::FileLock;
+use crate::volume::BACKUP_DIR;
+use crate::{Error, ErrorKind, ResultExt};
+
+pub struct BackupMountGuard<G: GenericMountGuard> {
+    backup_disk_mount_guard: Option<G>,
+    encrypted_guard: Option<TmpMountGuard>,
+    enc_key: String,
+    pub unencrypted_metadata: EmbassyOsRecoveryInfo,
+    pub metadata: BackupInfo,
+}
+impl<G: GenericMountGuard> BackupMountGuard<G> {
+    fn backup_disk_path(&self) -> &Path {
+        if let Some(guard) = &self.backup_disk_mount_guard {
+            guard.as_ref()
+        } else {
+            unreachable!()
+        }
+    }
+
+    #[instrument(skip_all)]
+    pub async fn mount(backup_disk_mount_guard: G, password: &str) -> Result<Self, Error> {
+        let backup_disk_path = backup_disk_mount_guard.as_ref();
+        let unencrypted_metadata_path =
+            backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor");
+        let mut unencrypted_metadata: EmbassyOsRecoveryInfo =
+            if tokio::fs::metadata(&unencrypted_metadata_path)
+                .await
+                .is_ok()
+            {
+                IoFormat::Cbor.from_slice(
+                    &tokio::fs::read(&unencrypted_metadata_path)
+                        .await
+                        .with_ctx(|_| {
+                            (
+                                crate::ErrorKind::Filesystem,
+                                unencrypted_metadata_path.display().to_string(),
+                            )
+                        })?,
+                )?
+            } else {
+                Default::default()
+            };
+        let enc_key = if let (Some(hash), Some(wrapped_key)) = (
+            unencrypted_metadata.password_hash.as_ref(),
+            unencrypted_metadata.wrapped_key.as_ref(),
+        ) {
+            let wrapped_key =
+                base32::decode(base32::Alphabet::RFC4648 { padding: true }, wrapped_key)
+                    .ok_or_else(|| {
+                        Error::new(
+                            eyre!("failed to decode wrapped key"),
+                            crate::ErrorKind::Backup,
+                        )
+                    })?;
+            check_password(hash, password)?;
+            String::from_utf8(decrypt_slice(wrapped_key, password))?
+        } else {
+            base32::encode(
+                base32::Alphabet::RFC4648 { padding: false },
+                &rand::random::<[u8; 32]>()[..],
+            )
+        };
+
+        if unencrypted_metadata.password_hash.is_none() {
+            unencrypted_metadata.password_hash = Some(
+                argon2::hash_encoded(
+                    password.as_bytes(),
+                    &rand::random::<[u8; 16]>()[..],
+                    &argon2::Config::rfc9106_low_mem(),
+                )
+                .with_kind(crate::ErrorKind::PasswordHashGeneration)?,
+            );
+        }
+        if unencrypted_metadata.wrapped_key.is_none() {
+            unencrypted_metadata.wrapped_key = Some(base32::encode(
+                base32::Alphabet::RFC4648 { padding: true },
+                &encrypt_slice(&enc_key, password),
+            ));
+        }
+
+        let crypt_path = backup_disk_path.join("EmbassyBackups/crypt");
+        if tokio::fs::metadata(&crypt_path).await.is_err() {
+            tokio::fs::create_dir_all(&crypt_path).await.with_ctx(|_| {
+                (
+                    crate::ErrorKind::Filesystem,
+                    crypt_path.display().to_string(),
+                )
+            })?;
+        }
+        let encrypted_guard =
+            TmpMountGuard::mount(&EcryptFS::new(&crypt_path, &enc_key), ReadWrite).await?;
+
+        let metadata_path = encrypted_guard.as_ref().join("metadata.cbor");
+        let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() {
+            IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
+                (
+                    crate::ErrorKind::Filesystem,
+                    metadata_path.display().to_string(),
+                )
+            })?)?
+        } else {
+            Default::default()
+        };
+
+        Ok(Self {
+            backup_disk_mount_guard: Some(backup_disk_mount_guard),
+            encrypted_guard: Some(encrypted_guard),
+            enc_key,
+            unencrypted_metadata,
+            metadata,
+        })
+    }
+
+    pub fn change_password(&mut self, new_password: &str) -> Result<(), Error> {
+        self.unencrypted_metadata.password_hash = Some(
+            argon2::hash_encoded(
+                new_password.as_bytes(),
+                &rand::random::<[u8; 16]>()[..],
+                &argon2::Config::rfc9106_low_mem(),
+            )
+            .with_kind(crate::ErrorKind::PasswordHashGeneration)?,
+        );
+        self.unencrypted_metadata.wrapped_key = Some(base32::encode(
+            base32::Alphabet::RFC4648 { padding: false },
+            &encrypt_slice(&self.enc_key, new_password),
+        ));
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn mount_package_backup(
+        &self,
+        id: &PackageId,
+    ) -> Result<PackageBackupMountGuard, Error> {
+        let lock = FileLock::new(Path::new(BACKUP_DIR).join(format!("{}.lock", id)), false).await?;
+        let mountpoint = Path::new(BACKUP_DIR).join(id);
+        bind(self.as_ref().join(id), &mountpoint, false).await?;
+        Ok(PackageBackupMountGuard {
+            mountpoint: Some(mountpoint),
+            lock: Some(lock),
+        })
+    }
+
+    #[instrument(skip_all)]
+    pub async fn save(&self) -> Result<(), Error> {
+        let metadata_path = self.as_ref().join("metadata.cbor");
+        let backup_disk_path = self.backup_disk_path();
+        let mut file = AtomicFile::new(&metadata_path, None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
+        file.write_all(&IoFormat::Cbor.to_vec(&self.metadata)?)
+            .await?;
+        file.save().await.with_kind(ErrorKind::Filesystem)?;
+        let unencrypted_metadata_path =
+            backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor");
+        let mut file = AtomicFile::new(&unencrypted_metadata_path, None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
+        file.write_all(&IoFormat::Cbor.to_vec(&self.unencrypted_metadata)?)
+            .await?;
+        file.save().await.with_kind(ErrorKind::Filesystem)?;
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn unmount(mut self) -> Result<(), Error> {
+        if let Some(guard) = self.encrypted_guard.take() {
+            guard.unmount().await?;
+        }
+        if let Some(guard) = self.backup_disk_mount_guard.take() {
+            guard.unmount().await?;
+        }
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn save_and_unmount(self) -> Result<(), Error> {
+        self.save().await?;
+        self.unmount().await?;
+        Ok(())
+    }
+}
+impl<G: GenericMountGuard> AsRef<Path> for BackupMountGuard<G> {
+    fn as_ref(&self) -> &Path {
+        if let Some(guard) = &self.encrypted_guard {
+            guard.as_ref()
+        } else {
+            unreachable!()
+        }
+    }
+}
+impl<G: GenericMountGuard> Drop for BackupMountGuard<G> {
+    fn drop(&mut self) {
+        let first = self.encrypted_guard.take();
+        let second = self.backup_disk_mount_guard.take();
+        tokio::spawn(async move {
+            if let Some(guard) = first {
+                guard.unmount().await.unwrap();
+            }
+            if let Some(guard) = second {
+                guard.unmount().await.unwrap();
+            }
+        });
+    }
+}
+
+pub struct PackageBackupMountGuard {
+    mountpoint: Option<PathBuf>,
+    lock: Option<FileLock>,
+}
+impl PackageBackupMountGuard {
+    pub async fn unmount(mut self) -> Result<(), Error> {
+        if let Some(mountpoint) = self.mountpoint.take() {
+            unmount(&mountpoint).await?;
+        }
+        if let Some(lock) = self.lock.take() {
+            lock.unlock().await?;
+        }
+        Ok(())
+    }
+}
+impl AsRef<Path> for PackageBackupMountGuard {
+    fn as_ref(&self) -> &Path {
+        if let Some(mountpoint) = &self.mountpoint {
+            mountpoint
+        } else {
+            unreachable!()
+        }
+    }
+}
+impl Drop for PackageBackupMountGuard {
+    fn drop(&mut self) {
+        let mountpoint = self.mountpoint.take();
+        let lock = self.lock.take();
+        tokio::spawn(async move {
+            if let Some(mountpoint) = mountpoint {
+                unmount(&mountpoint).await.unwrap();
+            }
+            if let Some(lock) = lock {
+                lock.unlock().await.unwrap();
+            }
+        });
+    }
+}

core/startos/src/disk/mount/filesystem/bind.rs

@@ -0,0 +1,54 @@
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType, ReadOnly};
+use crate::disk::mount::util::bind;
+use crate::{Error, ResultExt};
+
+pub struct Bind<SrcDir: AsRef<Path>> {
+    src_dir: SrcDir,
+}
+impl<SrcDir: AsRef<Path>> Bind<SrcDir> {
+    pub fn new(src_dir: SrcDir) -> Self {
+        Self { src_dir }
+    }
+}
+#[async_trait]
+impl<SrcDir: AsRef<Path> + Send + Sync> FileSystem for Bind<SrcDir> {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        bind(
+            self.src_dir.as_ref(),
+            mountpoint,
+            matches!(mount_type, ReadOnly),
+        )
+        .await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("Bind");
+        sha.update(
+            tokio::fs::canonicalize(self.src_dir.as_ref())
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        self.src_dir.as_ref().display().to_string(),
+                    )
+                })?
+                .as_os_str()
+                .as_bytes(),
+        );
+        Ok(sha.finalize())
+    }
+}

core/startos/src/disk/mount/filesystem/block_dev.rs

@@ -0,0 +1,67 @@
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use serde::{Deserialize, Serialize};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType, ReadOnly};
+use crate::util::Invoke;
+use crate::{Error, ResultExt};
+
+pub async fn mount(
+    logicalname: impl AsRef<Path>,
+    mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
+) -> Result<(), Error> {
+    tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
+    let mut cmd = tokio::process::Command::new("mount");
+    cmd.arg(logicalname.as_ref()).arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
+    Ok(())
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct BlockDev<LogicalName: AsRef<Path>> {
+    logicalname: LogicalName,
+}
+impl<LogicalName: AsRef<Path>> BlockDev<LogicalName> {
+    pub fn new(logicalname: LogicalName) -> Self {
+        BlockDev { logicalname }
+    }
+}
+#[async_trait]
+impl<LogicalName: AsRef<Path> + Send + Sync> FileSystem for BlockDev<LogicalName> {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount(self.logicalname.as_ref(), mountpoint, mount_type).await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("BlockDev");
+        sha.update(
+            tokio::fs::canonicalize(self.logicalname.as_ref())
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        self.logicalname.as_ref().display().to_string(),
+                    )
+                })?
+                .as_os_str()
+                .as_bytes(),
+        );
+        Ok(sha.finalize())
+    }
+}

core/startos/src/disk/mount/filesystem/cifs.rs

@@ -0,0 +1,107 @@
+use std::net::IpAddr;
+use std::os::unix::ffi::OsStrExt;
+use std::path::{Path, PathBuf};
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use serde::{Deserialize, Serialize};
+use sha2::Sha256;
+use tokio::process::Command;
+use tracing::instrument;
+
+use super::{FileSystem, MountType, ReadOnly};
+use crate::disk::mount::guard::TmpMountGuard;
+use crate::util::Invoke;
+use crate::Error;
+
+async fn resolve_hostname(hostname: &str) -> Result<IpAddr, Error> {
+    if let Ok(addr) = hostname.parse() {
+        return Ok(addr);
+    }
+    if hostname.ends_with(".local") {
+        return Ok(IpAddr::V4(crate::net::mdns::resolve_mdns(hostname).await?));
+    }
+    Ok(String::from_utf8(
+        Command::new("nmblookup")
+            .arg(hostname)
+            .invoke(crate::ErrorKind::Network)
+            .await?,
+    )?
+    .split(" ")
+    .next()
+    .unwrap()
+    .trim()
+    .parse()?)
+}
+
+#[instrument(skip_all)]
+pub async fn mount_cifs(
+    hostname: &str,
+    path: impl AsRef<Path>,
+    username: &str,
+    password: Option<&str>,
+    mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
+) -> Result<(), Error> {
+    tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
+    let ip: IpAddr = resolve_hostname(hostname).await?;
+    let absolute_path = Path::new("/").join(path.as_ref());
+    let mut cmd = Command::new("mount");
+    cmd.arg("-t")
+        .arg("cifs")
+        .env("USER", username)
+        .env("PASSWD", password.unwrap_or_default())
+        .arg(format!("//{}{}", ip, absolute_path.display()))
+        .arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro,noserverino");
+    } else {
+        cmd.arg("-o").arg("noserverino");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
+    Ok(())
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Cifs {
+    pub hostname: String,
+    pub path: PathBuf,
+    pub username: String,
+    pub password: Option<String>,
+}
+impl Cifs {
+    pub async fn mountable(&self) -> Result<(), Error> {
+        let guard = TmpMountGuard::mount(self, ReadOnly).await?;
+        guard.unmount().await?;
+        Ok(())
+    }
+}
+#[async_trait]
+impl FileSystem for Cifs {
+    async fn mount<P: AsRef<std::path::Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount_cifs(
+            &self.hostname,
+            &self.path,
+            &self.username,
+            self.password.as_ref().map(|p| p.as_str()),
+            mountpoint,
+            mount_type,
+        )
+        .await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("Cifs");
+        sha.update(self.hostname.as_bytes());
+        sha.update(self.path.as_os_str().as_bytes());
+        Ok(sha.finalize())
+    }
+}

core/startos/src/disk/mount/filesystem/ecryptfs.rs

@@ -0,0 +1,71 @@
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType};
+use crate::util::Invoke;
+use crate::{Error, ResultExt};
+
+pub async fn mount_ecryptfs<P0: AsRef<Path>, P1: AsRef<Path>>(
+    src: P0,
+    dst: P1,
+    key: &str,
+) -> Result<(), Error> {
+    tokio::fs::create_dir_all(dst.as_ref()).await?;
+    tokio::process::Command::new("mount")
+        .arg("-t")
+        .arg("ecryptfs")
+        .arg(src.as_ref())
+        .arg(dst.as_ref())
+        .arg("-o")
+        // for more information `man ecryptfs` 
+        .arg(format!("key=passphrase:passphrase_passwd={},ecryptfs_cipher=aes,ecryptfs_key_bytes=32,ecryptfs_passthrough=n,ecryptfs_enable_filename_crypto=y,no_sig_cache", key))
+        .input(Some(&mut std::io::Cursor::new(b"\n")))
+        .invoke(crate::ErrorKind::Filesystem).await?;
+    Ok(())
+}
+
+pub struct EcryptFS<EncryptedDir: AsRef<Path>, Key: AsRef<str>> {
+    encrypted_dir: EncryptedDir,
+    key: Key,
+}
+impl<EncryptedDir: AsRef<Path>, Key: AsRef<str>> EcryptFS<EncryptedDir, Key> {
+    pub fn new(encrypted_dir: EncryptedDir, key: Key) -> Self {
+        EcryptFS { encrypted_dir, key }
+    }
+}
+#[async_trait]
+impl<EncryptedDir: AsRef<Path> + Send + Sync, Key: AsRef<str> + Send + Sync> FileSystem
+    for EcryptFS<EncryptedDir, Key>
+{
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        _mount_type: MountType, // ignored - inherited from parent fs
+    ) -> Result<(), Error> {
+        mount_ecryptfs(self.encrypted_dir.as_ref(), mountpoint, self.key.as_ref()).await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("EcryptFS");
+        sha.update(
+            tokio::fs::canonicalize(self.encrypted_dir.as_ref())
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        self.encrypted_dir.as_ref().display().to_string(),
+                    )
+                })?
+                .as_os_str()
+                .as_bytes(),
+        );
+        Ok(sha.finalize())
+    }
+}

core/startos/src/disk/mount/filesystem/efivarfs.rs

@@ -0,0 +1,39 @@
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType, ReadOnly};
+use crate::util::Invoke;
+use crate::Error;
+
+pub struct EfiVarFs;
+#[async_trait]
+impl FileSystem for EfiVarFs {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
+        let mut cmd = tokio::process::Command::new("mount");
+        cmd.arg("-t")
+            .arg("efivarfs")
+            .arg("efivarfs")
+            .arg(mountpoint.as_ref());
+        if mount_type == ReadOnly {
+            cmd.arg("-o").arg("ro");
+        }
+        cmd.invoke(crate::ErrorKind::Filesystem).await?;
+        Ok(())
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("EfiVarFs");
+        Ok(sha.finalize())
+    }
+}

core/startos/src/disk/mount/filesystem/httpdirfs.rs

@@ -0,0 +1,52 @@
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use reqwest::Url;
+use serde::{Deserialize, Serialize};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType};
+use crate::util::Invoke;
+use crate::Error;
+
+pub async fn mount_httpdirfs(url: &Url, mountpoint: impl AsRef<Path>) -> Result<(), Error> {
+    tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
+    let mut cmd = tokio::process::Command::new("httpdirfs");
+    cmd.arg("--cache")
+        .arg("--single-file-mode")
+        .arg(url.as_str())
+        .arg(mountpoint.as_ref());
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
+    Ok(())
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct HttpDirFS {
+    url: Url,
+}
+impl HttpDirFS {
+    pub fn new(url: Url) -> Self {
+        HttpDirFS { url }
+    }
+}
+#[async_trait]
+impl FileSystem for HttpDirFS {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        _mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount_httpdirfs(&self.url, mountpoint).await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("HttpDirFS");
+        sha.update(self.url.as_str());
+        Ok(sha.finalize())
+    }
+}

core/startos/src/disk/mount/filesystem/label.rs

@@ -0,0 +1,52 @@
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType, ReadOnly};
+use crate::util::Invoke;
+use crate::Error;
+
+pub async fn mount_label(
+    label: &str,
+    mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
+) -> Result<(), Error> {
+    tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
+    let mut cmd = tokio::process::Command::new("mount");
+    cmd.arg("-L").arg(label).arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
+    Ok(())
+}
+
+pub struct Label<S: AsRef<str>> {
+    label: S,
+}
+impl<S: AsRef<str>> Label<S> {
+    pub fn new(label: S) -> Self {
+        Label { label }
+    }
+}
+#[async_trait]
+impl<S: AsRef<str> + Send + Sync> FileSystem for Label<S> {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount_label(self.label.as_ref(), mountpoint, mount_type).await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("Label");
+        sha.update(self.label.as_ref().as_bytes());
+        Ok(sha.finalize())
+    }
+}

core/startos/src/disk/mount/filesystem/mod.rs

@@ -0,0 +1,36 @@
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::OutputSizeUser;
+use sha2::Sha256;
+
+use crate::Error;
+
+pub mod bind;
+pub mod block_dev;
+pub mod cifs;
+pub mod ecryptfs;
+pub mod efivarfs;
+pub mod httpdirfs;
+pub mod label;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum MountType {
+    ReadOnly,
+    ReadWrite,
+}
+
+pub use MountType::*;
+
+#[async_trait]
+pub trait FileSystem {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error>;
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error>;
+}

core/startos/src/disk/mount/guard.rs

@@ -0,0 +1,142 @@
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Weak};
+
+use lazy_static::lazy_static;
+use models::ResultExt;
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+use super::filesystem::{FileSystem, MountType, ReadOnly, ReadWrite};
+use super::util::unmount;
+use crate::util::Invoke;
+use crate::Error;
+
+pub const TMP_MOUNTPOINT: &'static str = "/media/embassy/tmp";
+
+#[async_trait::async_trait]
+pub trait GenericMountGuard: AsRef<Path> + std::fmt::Debug + Send + Sync + 'static {
+    async fn unmount(mut self) -> Result<(), Error>;
+}
+
+#[derive(Debug)]
+pub struct MountGuard {
+    mountpoint: PathBuf,
+    mounted: bool,
+}
+impl MountGuard {
+    pub async fn mount(
+        filesystem: &impl FileSystem,
+        mountpoint: impl AsRef<Path>,
+        mount_type: MountType,
+    ) -> Result<Self, Error> {
+        let mountpoint = mountpoint.as_ref().to_owned();
+        filesystem.mount(&mountpoint, mount_type).await?;
+        Ok(MountGuard {
+            mountpoint,
+            mounted: true,
+        })
+    }
+    pub async fn unmount(mut self, delete_mountpoint: bool) -> Result<(), Error> {
+        if self.mounted {
+            unmount(&self.mountpoint).await?;
+            if delete_mountpoint {
+                match tokio::fs::remove_dir(&self.mountpoint).await {
+                    Err(e) if e.raw_os_error() == Some(39) => Ok(()), // directory not empty
+                    a => a,
+                }
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        format!("rm {}", self.mountpoint.display()),
+                    )
+                })?;
+            }
+            self.mounted = false;
+        }
+        Ok(())
+    }
+}
+impl AsRef<Path> for MountGuard {
+    fn as_ref(&self) -> &Path {
+        &self.mountpoint
+    }
+}
+impl Drop for MountGuard {
+    fn drop(&mut self) {
+        if self.mounted {
+            let mountpoint = std::mem::take(&mut self.mountpoint);
+            tokio::spawn(async move { unmount(mountpoint).await.unwrap() });
+        }
+    }
+}
+#[async_trait::async_trait]
+impl GenericMountGuard for MountGuard {
+    async fn unmount(mut self) -> Result<(), Error> {
+        MountGuard::unmount(self, false).await
+    }
+}
+
+async fn tmp_mountpoint(source: &impl FileSystem) -> Result<PathBuf, Error> {
+    Ok(Path::new(TMP_MOUNTPOINT).join(base32::encode(
+        base32::Alphabet::RFC4648 { padding: false },
+        &source.source_hash().await?,
+    )))
+}
+
+lazy_static! {
+    static ref TMP_MOUNTS: Mutex<BTreeMap<PathBuf, (MountType, Weak<MountGuard>)>> =
+        Mutex::new(BTreeMap::new());
+}
+
+#[derive(Debug)]
+pub struct TmpMountGuard {
+    guard: Arc<MountGuard>,
+}
+impl TmpMountGuard {
+    /// DRAGONS: if you try to mount something as ro and rw at the same time, the ro mount will be upgraded to rw.
+    #[instrument(skip_all)]
+    pub async fn mount(filesystem: &impl FileSystem, mount_type: MountType) -> Result<Self, Error> {
+        let mountpoint = tmp_mountpoint(filesystem).await?;
+        let mut tmp_mounts = TMP_MOUNTS.lock().await;
+        if !tmp_mounts.contains_key(&mountpoint) {
+            tmp_mounts.insert(mountpoint.clone(), (mount_type, Weak::new()));
+        }
+        let (prev_mt, weak_slot) = tmp_mounts.get_mut(&mountpoint).unwrap();
+        if let Some(guard) = weak_slot.upgrade() {
+            // upgrade to rw
+            if *prev_mt == ReadOnly && mount_type == ReadWrite {
+                tokio::process::Command::new("mount")
+                    .arg("-o")
+                    .arg("remount,rw")
+                    .arg(&mountpoint)
+                    .invoke(crate::ErrorKind::Filesystem)
+                    .await?;
+                *prev_mt = ReadWrite;
+            }
+            Ok(TmpMountGuard { guard })
+        } else {
+            let guard = Arc::new(MountGuard::mount(filesystem, &mountpoint, mount_type).await?);
+            *weak_slot = Arc::downgrade(&guard);
+            *prev_mt = mount_type;
+            Ok(TmpMountGuard { guard })
+        }
+    }
+    pub async fn unmount(self) -> Result<(), Error> {
+        if let Ok(guard) = Arc::try_unwrap(self.guard) {
+            guard.unmount(true).await?;
+        }
+        Ok(())
+    }
+}
+impl AsRef<Path> for TmpMountGuard {
+    fn as_ref(&self) -> &Path {
+        (&*self.guard).as_ref()
+    }
+}
+#[async_trait::async_trait]
+impl GenericMountGuard for TmpMountGuard {
+    async fn unmount(mut self) -> Result<(), Error> {
+        TmpMountGuard::unmount(self).await
+    }
+}

core/startos/src/disk/mount/mod.rs

@@ -0,0 +1,4 @@
+pub mod backup;
+pub mod filesystem;
+pub mod guard;
+pub mod util;

core/startos/src/disk/mount/util.rs

@@ -0,0 +1,52 @@
+use std::path::Path;
+
+use tracing::instrument;
+
+use crate::util::Invoke;
+use crate::Error;
+
+#[instrument(skip_all)]
+pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(
+    src: P0,
+    dst: P1,
+    read_only: bool,
+) -> Result<(), Error> {
+    tracing::info!(
+        "Binding {} to {}",
+        src.as_ref().display(),
+        dst.as_ref().display()
+    );
+    let is_mountpoint = tokio::process::Command::new("mountpoint")
+        .arg(dst.as_ref())
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .status()
+        .await?;
+    if is_mountpoint.success() {
+        unmount(dst.as_ref()).await?;
+    }
+    tokio::fs::create_dir_all(&src).await?;
+    tokio::fs::create_dir_all(&dst).await?;
+    let mut mount_cmd = tokio::process::Command::new("mount");
+    mount_cmd.arg("--bind");
+    if read_only {
+        mount_cmd.arg("-o").arg("ro");
+    }
+    mount_cmd
+        .arg(src.as_ref())
+        .arg(dst.as_ref())
+        .invoke(crate::ErrorKind::Filesystem)
+        .await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn unmount<P: AsRef<Path>>(mountpoint: P) -> Result<(), Error> {
+    tracing::debug!("Unmounting {}.", mountpoint.as_ref().display());
+    tokio::process::Command::new("umount")
+        .arg("-l")
+        .arg(mountpoint.as_ref())
+        .invoke(crate::ErrorKind::Filesystem)
+        .await?;
+    Ok(())
+}

core/startos/src/disk/util.rs

@@ -0,0 +1,489 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+
+use color_eyre::eyre::{self, eyre};
+use futures::TryStreamExt;
+use nom::bytes::complete::{tag, take_till1};
+use nom::character::complete::multispace1;
+use nom::character::is_space;
+use nom::combinator::{opt, rest};
+use nom::sequence::{pair, preceded, terminated};
+use nom::IResult;
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+use tokio::process::Command;
+use tracing::instrument;
+
+use super::mount::filesystem::block_dev::BlockDev;
+use super::mount::filesystem::ReadOnly;
+use super::mount::guard::TmpMountGuard;
+use crate::disk::OsPartitionInfo;
+use crate::util::serde::IoFormat;
+use crate::util::{Invoke, Version};
+use crate::{Error, ResultExt as _};
+
+#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum PartitionTable {
+    Mbr,
+    Gpt,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DiskInfo {
+    pub logicalname: PathBuf,
+    pub partition_table: Option<PartitionTable>,
+    pub vendor: Option<String>,
+    pub model: Option<String>,
+    pub partitions: Vec<PartitionInfo>,
+    pub capacity: u64,
+    pub guid: Option<String>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct PartitionInfo {
+    pub logicalname: PathBuf,
+    pub label: Option<String>,
+    pub capacity: u64,
+    pub used: Option<u64>,
+    pub embassy_os: Option<EmbassyOsRecoveryInfo>,
+    pub guid: Option<String>,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct EmbassyOsRecoveryInfo {
+    pub version: Version,
+    pub full: bool,
+    pub password_hash: Option<String>,
+    pub wrapped_key: Option<String>,
+}
+
+const DISK_PATH: &str = "/dev/disk/by-path";
+const SYS_BLOCK_PATH: &str = "/sys/block";
+
+lazy_static::lazy_static! {
+    static ref PARTITION_REGEX: Regex = Regex::new("-part[0-9]+$").unwrap();
+}
+
+#[instrument(skip_all)]
+pub async fn get_partition_table<P: AsRef<Path>>(path: P) -> Result<Option<PartitionTable>, Error> {
+    Ok(String::from_utf8(
+        Command::new("fdisk")
+            .arg("-l")
+            .arg(path.as_ref())
+            .invoke(crate::ErrorKind::BlockDevice)
+            .await?,
+    )?
+    .lines()
+    .find_map(|l| l.strip_prefix("Disklabel type:"))
+    .and_then(|t| match t.trim() {
+        "dos" => Some(PartitionTable::Mbr),
+        "gpt" => Some(PartitionTable::Gpt),
+        _ => None,
+    }))
+}
+
+#[instrument(skip_all)]
+pub async fn get_vendor<P: AsRef<Path>>(path: P) -> Result<Option<String>, Error> {
+    let vendor = tokio::fs::read_to_string(
+        Path::new(SYS_BLOCK_PATH)
+            .join(path.as_ref().strip_prefix("/dev").map_err(|_| {
+                Error::new(
+                    eyre!("not a canonical block device"),
+                    crate::ErrorKind::BlockDevice,
+                )
+            })?)
+            .join("device")
+            .join("vendor"),
+    )
+    .await?
+    .trim()
+    .to_owned();
+    Ok(if vendor.is_empty() {
+        None
+    } else {
+        Some(vendor)
+    })
+}
+
+#[instrument(skip_all)]
+pub async fn get_model<P: AsRef<Path>>(path: P) -> Result<Option<String>, Error> {
+    let model = tokio::fs::read_to_string(
+        Path::new(SYS_BLOCK_PATH)
+            .join(path.as_ref().strip_prefix("/dev").map_err(|_| {
+                Error::new(
+                    eyre!("not a canonical block device"),
+                    crate::ErrorKind::BlockDevice,
+                )
+            })?)
+            .join("device")
+            .join("model"),
+    )
+    .await?
+    .trim()
+    .to_owned();
+    Ok(if model.is_empty() { None } else { Some(model) })
+}
+
+#[instrument(skip_all)]
+pub async fn get_capacity<P: AsRef<Path>>(path: P) -> Result<u64, Error> {
+    Ok(String::from_utf8(
+        Command::new("blockdev")
+            .arg("--getsize64")
+            .arg(path.as_ref())
+            .invoke(crate::ErrorKind::BlockDevice)
+            .await?,
+    )?
+    .trim()
+    .parse::<u64>()?)
+}
+
+#[instrument(skip_all)]
+pub async fn get_label<P: AsRef<Path>>(path: P) -> Result<Option<String>, Error> {
+    let label = String::from_utf8(
+        Command::new("lsblk")
+            .arg("-no")
+            .arg("label")
+            .arg(path.as_ref())
+            .invoke(crate::ErrorKind::BlockDevice)
+            .await?,
+    )?
+    .trim()
+    .to_owned();
+    Ok(if label.is_empty() { None } else { Some(label) })
+}
+
+#[instrument(skip_all)]
+pub async fn get_used<P: AsRef<Path>>(path: P) -> Result<u64, Error> {
+    Ok(String::from_utf8(
+        Command::new("df")
+            .arg("--output=used")
+            .arg("--block-size=1")
+            .arg(path.as_ref())
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?,
+    )?
+    .lines()
+    .skip(1)
+    .next()
+    .unwrap_or_default()
+    .trim()
+    .parse::<u64>()?)
+}
+
+#[instrument(skip_all)]
+pub async fn get_available<P: AsRef<Path>>(path: P) -> Result<u64, Error> {
+    Ok(String::from_utf8(
+        Command::new("df")
+            .arg("--output=avail")
+            .arg("--block-size=1")
+            .arg(path.as_ref())
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?,
+    )?
+    .lines()
+    .skip(1)
+    .next()
+    .unwrap_or_default()
+    .trim()
+    .parse::<u64>()?)
+}
+
+#[instrument(skip_all)]
+pub async fn get_percentage<P: AsRef<Path>>(path: P) -> Result<u64, Error> {
+    Ok(String::from_utf8(
+        Command::new("df")
+            .arg("--output=pcent")
+            .arg(path.as_ref())
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?,
+    )?
+    .lines()
+    .skip(1)
+    .next()
+    .unwrap_or_default()
+    .trim()
+    .strip_suffix("%")
+    .unwrap()
+    .parse::<u64>()?)
+}
+
+#[instrument(skip_all)]
+pub async fn pvscan() -> Result<BTreeMap<PathBuf, Option<String>>, Error> {
+    let pvscan_out = Command::new("pvscan")
+        .invoke(crate::ErrorKind::DiskManagement)
+        .await?;
+    let pvscan_out_str = std::str::from_utf8(&pvscan_out)?;
+    Ok(parse_pvscan_output(pvscan_out_str))
+}
+
+pub async fn recovery_info(
+    mountpoint: impl AsRef<Path>,
+) -> Result<Option<EmbassyOsRecoveryInfo>, Error> {
+    let backup_unencrypted_metadata_path = mountpoint
+        .as_ref()
+        .join("EmbassyBackups/unencrypted-metadata.cbor");
+    if tokio::fs::metadata(&backup_unencrypted_metadata_path)
+        .await
+        .is_ok()
+    {
+        return Ok(Some(
+            IoFormat::Cbor.from_slice(
+                &tokio::fs::read(&backup_unencrypted_metadata_path)
+                    .await
+                    .with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            backup_unencrypted_metadata_path.display().to_string(),
+                        )
+                    })?,
+            )?,
+        ));
+    }
+
+    Ok(None)
+}
+
+#[instrument(skip_all)]
+pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
+    struct DiskIndex {
+        parts: BTreeSet<PathBuf>,
+        internal: bool,
+    }
+    let disk_guids = pvscan().await?;
+    let disks = tokio_stream::wrappers::ReadDirStream::new(
+        tokio::fs::read_dir(DISK_PATH)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, DISK_PATH))?,
+    )
+    .map_err(|e| {
+        Error::new(
+            eyre::Error::from(e).wrap_err(DISK_PATH),
+            crate::ErrorKind::Filesystem,
+        )
+    })
+    .try_fold(
+        BTreeMap::<PathBuf, DiskIndex>::new(),
+        |mut disks, dir_entry| async move {
+            if let Some(disk_path) = dir_entry.path().file_name().and_then(|s| s.to_str()) {
+                let (disk_path, part_path) = if let Some(end) = PARTITION_REGEX.find(disk_path) {
+                    (
+                        disk_path.strip_suffix(end.as_str()).unwrap_or_default(),
+                        Some(disk_path),
+                    )
+                } else {
+                    (disk_path, None)
+                };
+                let disk_path = Path::new(DISK_PATH).join(disk_path);
+                let disk = tokio::fs::canonicalize(&disk_path).await.with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        disk_path.display().to_string(),
+                    )
+                })?;
+                let part = if let Some(part_path) = part_path {
+                    let part_path = Path::new(DISK_PATH).join(part_path);
+                    let part = tokio::fs::canonicalize(&part_path).await.with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            part_path.display().to_string(),
+                        )
+                    })?;
+                    Some(part)
+                } else {
+                    None
+                };
+                if !disks.contains_key(&disk) {
+                    disks.insert(
+                        disk.clone(),
+                        DiskIndex {
+                            parts: BTreeSet::new(),
+                            internal: false,
+                        },
+                    );
+                }
+                if let Some(part) = part {
+                    if os.contains(&part) {
+                        disks.get_mut(&disk).unwrap().internal = true;
+                    } else {
+                        disks.get_mut(&disk).unwrap().parts.insert(part);
+                    }
+                }
+            }
+            Ok(disks)
+        },
+    )
+    .await?;
+
+    let mut res = Vec::with_capacity(disks.len());
+    for (disk, index) in disks {
+        if index.internal {
+            for part in index.parts {
+                let mut disk_info = disk_info(disk.clone()).await;
+                let part_info = part_info(part).await;
+                disk_info.logicalname = part_info.logicalname.clone();
+                disk_info.capacity = part_info.capacity;
+                if let Some(g) = disk_guids.get(&disk_info.logicalname) {
+                    disk_info.guid = g.clone();
+                } else {
+                    disk_info.partitions = vec![part_info];
+                }
+                res.push(disk_info);
+            }
+        } else {
+            let mut disk_info = disk_info(disk).await;
+            disk_info.partitions = Vec::with_capacity(index.parts.len());
+            if let Some(g) = disk_guids.get(&disk_info.logicalname) {
+                disk_info.guid = g.clone();
+            } else {
+                for part in index.parts {
+                    let mut part_info = part_info(part).await;
+                    if let Some(g) = disk_guids.get(&part_info.logicalname) {
+                        part_info.guid = g.clone();
+                    }
+                    disk_info.partitions.push(part_info);
+                }
+            }
+            res.push(disk_info);
+        }
+    }
+
+    Ok(res)
+}
+
+async fn disk_info(disk: PathBuf) -> DiskInfo {
+    let partition_table = get_partition_table(&disk)
+        .await
+        .map_err(|e| {
+            tracing::warn!(
+                "Could not get partition table of {}: {}",
+                disk.display(),
+                e.source
+            )
+        })
+        .unwrap_or_default();
+    let vendor = get_vendor(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get vendor of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    let model = get_model(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get model of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    let capacity = get_capacity(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    DiskInfo {
+        logicalname: disk,
+        partition_table,
+        vendor,
+        model,
+        partitions: Vec::new(),
+        capacity,
+        guid: None,
+    }
+}
+
+async fn part_info(part: PathBuf) -> PartitionInfo {
+    let mut embassy_os = None;
+    let label = get_label(&part)
+        .await
+        .map_err(|e| tracing::warn!("Could not get label of {}: {}", part.display(), e.source))
+        .unwrap_or_default();
+    let capacity = get_capacity(&part)
+        .await
+        .map_err(|e| tracing::warn!("Could not get capacity of {}: {}", part.display(), e.source))
+        .unwrap_or_default();
+    let mut used = None;
+
+    match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await {
+        Err(e) => tracing::warn!("Could not collect usage information: {}", e.source),
+        Ok(mount_guard) => {
+            used = get_used(&mount_guard)
+                .await
+                .map_err(|e| {
+                    tracing::warn!("Could not get usage of {}: {}", part.display(), e.source)
+                })
+                .ok();
+            if let Some(recovery_info) = match recovery_info(&mount_guard).await {
+                Ok(a) => a,
+                Err(e) => {
+                    tracing::error!("Error fetching unencrypted backup metadata: {}", e);
+                    None
+                }
+            } {
+                embassy_os = Some(recovery_info)
+            }
+            if let Err(e) = mount_guard.unmount().await {
+                tracing::error!("Error unmounting partition {}: {}", part.display(), e);
+            }
+        }
+    }
+
+    PartitionInfo {
+        logicalname: part,
+        label,
+        capacity,
+        used,
+        embassy_os,
+        guid: None,
+    }
+}
+
+fn parse_pvscan_output(pvscan_output: &str) -> BTreeMap<PathBuf, Option<String>> {
+    fn parse_line(line: &str) -> IResult<&str, (&str, Option<&str>)> {
+        let pv_parse = preceded(
+            tag("  PV "),
+            terminated(take_till1(|c| is_space(c as u8)), multispace1),
+        );
+        let vg_parse = preceded(
+            opt(tag("is in exported ")),
+            preceded(
+                tag("VG "),
+                terminated(take_till1(|c| is_space(c as u8)), multispace1),
+            ),
+        );
+        let mut parser = terminated(pair(pv_parse, opt(vg_parse)), rest);
+        parser(line)
+    }
+    let lines = pvscan_output.lines();
+    let n = lines.clone().count();
+    let entries = lines.take(n.saturating_sub(1));
+    let mut ret = BTreeMap::new();
+    for entry in entries {
+        match parse_line(entry) {
+            Ok((_, (pv, vg))) => {
+                ret.insert(PathBuf::from(pv), vg.map(|s| s.to_owned()));
+            }
+            Err(_) => {
+                tracing::warn!("Failed to parse pvscan output line: {}", entry);
+            }
+        }
+    }
+    ret
+}
+
+#[test]
+fn test_pvscan_parser() {
+    let s1 = r#"  PV /dev/mapper/cryptdata   VG data            lvm2 [1.81 TiB / 0    free]
+  PV /dev/sdb                                   lvm2 [931.51 GiB]
+  Total: 2 [2.72 TiB] / in use: 1 [1.81 TiB] / in no VG: 1 [931.51 GiB]
+"#;
+    let s2 = r#"  PV /dev/sdb   VG EMBASSY_LZHJAENWGPCJJL6C6AXOD7OOOIJG7HFBV4GYRJH6HADXUCN4BRWQ   lvm2 [931.51 GiB / 0    free]
+  Total: 1 [931.51 GiB] / in use: 1 [931.51 GiB] / in no VG: 0 [0   ]
+"#;
+    let s3 = r#"  PV /dev/mapper/cryptdata   VG data            lvm2 [1.81 TiB / 0    free]
+  Total: 1 [1.81 TiB] / in use: 1 [1.81 TiB] / in no VG: 0 [0   ]
+"#;
+    let s4 = r#"  PV /dev/sda    is in exported VG EMBASSY_ZFHOCTYV3ZJMJW3OTFMG55LSQZLP667EDNZKDNUJKPJX5HE6S5HQ [931.51 GiB / 0    free]
+  Total: 1 [931.51 GiB] / in use: 1 [931.51 GiB] / in no VG: 0 [0   ]
+"#;
+    println!("{:?}", parse_pvscan_output(s1));
+    println!("{:?}", parse_pvscan_output(s2));
+    println!("{:?}", parse_pvscan_output(s3));
+    println!("{:?}", parse_pvscan_output(s4));
+}

core/startos/src/error.rs

@@ -0,0 +1,60 @@
+use color_eyre::eyre::eyre;
+pub use models::{Error, ErrorKind, OptionExt, ResultExt};
+
+#[derive(Debug, Default)]
+pub struct ErrorCollection(Vec<Error>);
+impl ErrorCollection {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn handle<T, E: Into<Error>>(&mut self, result: Result<T, E>) -> Option<T> {
+        match result {
+            Ok(a) => Some(a),
+            Err(e) => {
+                self.0.push(e.into());
+                None
+            }
+        }
+    }
+
+    pub fn into_result(self) -> Result<(), Error> {
+        if self.0.is_empty() {
+            Ok(())
+        } else {
+            Err(Error::new(eyre!("{}", self), ErrorKind::MultipleErrors))
+        }
+    }
+}
+impl From<ErrorCollection> for Result<(), Error> {
+    fn from(e: ErrorCollection) -> Self {
+        e.into_result()
+    }
+}
+impl<T, E: Into<Error>> Extend<Result<T, E>> for ErrorCollection {
+    fn extend<I: IntoIterator<Item = Result<T, E>>>(&mut self, iter: I) {
+        for item in iter {
+            self.handle(item);
+        }
+    }
+}
+impl std::fmt::Display for ErrorCollection {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for (idx, e) in self.0.iter().enumerate() {
+            if idx > 0 {
+                write!(f, "; ")?;
+            }
+            write!(f, "{}", e)?;
+        }
+        Ok(())
+    }
+}
+
+#[macro_export]
+macro_rules! ensure_code {
+    ($x:expr, $c:expr, $fmt:expr $(, $arg:expr)*) => {
+        if !($x) {
+            return Err(crate::error::Error::new(color_eyre::eyre::eyre!($fmt, $($arg, )*), $c));
+        }
+    };
+}

core/startos/src/firmware.rs

@@ -0,0 +1,70 @@
+use std::path::Path;
+
+use async_compression::tokio::bufread::GzipDecoder;
+use tokio::fs::File;
+use tokio::io::{AsyncRead, BufReader};
+use tokio::process::Command;
+
+use crate::disk::fsck::RequiresReboot;
+use crate::prelude::*;
+use crate::util::Invoke;
+
+pub async fn update_firmware() -> Result<RequiresReboot, Error> {
+    let product_name = String::from_utf8(
+        Command::new("dmidecode")
+            .arg("-s")
+            .arg("system-product-name")
+            .invoke(ErrorKind::Firmware)
+            .await?,
+    )?
+    .trim()
+    .to_owned();
+    if product_name.is_empty() {
+        return Ok(RequiresReboot(false));
+    }
+    let firmware_dir = Path::new("/usr/lib/startos/firmware").join(&product_name);
+    if tokio::fs::metadata(&firmware_dir).await.is_ok() {
+        let current_firmware = String::from_utf8(
+            Command::new("dmidecode")
+                .arg("-s")
+                .arg("bios-version")
+                .invoke(ErrorKind::Firmware)
+                .await?,
+        )?
+        .trim()
+        .to_owned();
+        if tokio::fs::metadata(firmware_dir.join(format!("{current_firmware}.rom.gz")))
+            .await
+            .is_err()
+            && tokio::fs::metadata(firmware_dir.join(format!("{current_firmware}.rom")))
+                .await
+                .is_err()
+        {
+            let mut firmware_read_dir = tokio::fs::read_dir(&firmware_dir).await?;
+            while let Some(entry) = firmware_read_dir.next_entry().await? {
+                let filename = entry.file_name().to_string_lossy().into_owned();
+                let rdr: Option<Box<dyn AsyncRead + Unpin + Send>> =
+                    if filename.ends_with(".rom.gz") {
+                        Some(Box::new(GzipDecoder::new(BufReader::new(
+                            File::open(entry.path()).await?,
+                        ))))
+                    } else if filename.ends_with(".rom") {
+                        Some(Box::new(File::open(entry.path()).await?))
+                    } else {
+                        None
+                    };
+                if let Some(mut rdr) = rdr {
+                    Command::new("flashrom")
+                        .arg("-p")
+                        .arg("internal")
+                        .arg("-w-")
+                        .input(Some(&mut rdr))
+                        .invoke(ErrorKind::Firmware)
+                        .await?;
+                    return Ok(RequiresReboot(true));
+                }
+            }
+        }
+    }
+    Ok(RequiresReboot(false))
+}

core/startos/src/hostname.rs

@@ -0,0 +1,83 @@
+use rand::{thread_rng, Rng};
+use tokio::process::Command;
+use tracing::instrument;
+
+use crate::util::Invoke;
+use crate::{Error, ErrorKind};
+#[derive(Clone, serde::Deserialize, serde::Serialize, Debug)]
+pub struct Hostname(pub String);
+
+lazy_static::lazy_static! {
+    static ref ADJECTIVES: Vec<String> = include_str!("./assets/adjectives.txt").lines().map(|x| x.to_string()).collect();
+    static ref NOUNS: Vec<String> = include_str!("./assets/nouns.txt").lines().map(|x| x.to_string()).collect();
+}
+impl AsRef<str> for Hostname {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
+}
+
+impl Hostname {
+    pub fn lan_address(&self) -> String {
+        format!("https://{}.local", self.0)
+    }
+
+    pub fn local_domain_name(&self) -> String {
+        format!("{}.local", self.0)
+    }
+    pub fn no_dot_host_name(&self) -> String {
+        self.0.to_owned()
+    }
+}
+
+pub fn generate_hostname() -> Hostname {
+    let mut rng = thread_rng();
+    let adjective = &ADJECTIVES[rng.gen_range(0..ADJECTIVES.len())];
+    let noun = &NOUNS[rng.gen_range(0..NOUNS.len())];
+    Hostname(format!("{adjective}-{noun}"))
+}
+
+pub fn generate_id() -> String {
+    let id = uuid::Uuid::new_v4();
+    id.to_string()
+}
+
+#[instrument(skip_all)]
+pub async fn get_current_hostname() -> Result<Hostname, Error> {
+    let out = Command::new("hostname")
+        .invoke(ErrorKind::ParseSysInfo)
+        .await?;
+    let out_string = String::from_utf8(out)?;
+    Ok(Hostname(out_string.trim().to_owned()))
+}
+
+#[instrument(skip_all)]
+pub async fn set_hostname(hostname: &Hostname) -> Result<(), Error> {
+    let hostname: &String = &hostname.0;
+    Command::new("hostnamectl")
+        .arg("--static")
+        .arg("set-hostname")
+        .arg(hostname)
+        .invoke(ErrorKind::ParseSysInfo)
+        .await?;
+    Command::new("sed")
+        .arg("-i")
+        .arg(format!(
+            "s/\\(\\s\\)localhost\\( {hostname}\\)\\?/\\1localhost {hostname}/g"
+        ))
+        .arg("/etc/hosts")
+        .invoke(ErrorKind::ParseSysInfo)
+        .await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn sync_hostname(hostname: &Hostname) -> Result<(), Error> {
+    set_hostname(hostname).await?;
+    Command::new("systemctl")
+        .arg("restart")
+        .arg("avahi-daemon")
+        .invoke(crate::ErrorKind::Network)
+        .await?;
+    Ok(())
+}

core/startos/src/init.rs

@@ -0,0 +1,455 @@
+use std::fs::Permissions;
+use std::os::unix::fs::PermissionsExt;
+use std::path::Path;
+use std::time::{Duration, SystemTime};
+
+use color_eyre::eyre::eyre;
+use helpers::NonDetachingJoinHandle;
+use models::ResultExt;
+use rand::random;
+use sqlx::{Pool, Postgres};
+use tokio::process::Command;
+use tracing::instrument;
+
+use crate::account::AccountInfo;
+use crate::context::rpc::RpcContextConfig;
+use crate::db::model::ServerStatus;
+use crate::disk::mount::util::unmount;
+use crate::install::PKG_ARCHIVE_DIR;
+use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
+use crate::prelude::*;
+use crate::sound::BEP;
+use crate::util::cpupower::{
+    current_governor, get_available_governors, set_governor, GOVERNOR_PERFORMANCE,
+};
+use crate::util::docker::{create_bridge_network, CONTAINER_DATADIR, CONTAINER_TOOL};
+use crate::util::Invoke;
+use crate::{Error, ARCH};
+
+pub const SYSTEM_REBUILD_PATH: &str = "/media/embassy/config/system-rebuild";
+pub const STANDBY_MODE_PATH: &str = "/media/embassy/config/standby";
+
+pub async fn check_time_is_synchronized() -> Result<bool, Error> {
+    Ok(String::from_utf8(
+        Command::new("timedatectl")
+            .arg("show")
+            .arg("-p")
+            .arg("NTPSynchronized")
+            .invoke(crate::ErrorKind::Unknown)
+            .await?,
+    )?
+    .trim()
+        == "NTPSynchronized=yes")
+}
+
+// must be idempotent
+#[tracing::instrument(skip_all)]
+pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
+    let db_dir = datadir.as_ref().join("main/postgresql");
+    if tokio::process::Command::new("mountpoint")
+        .arg("/var/lib/postgresql")
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .status()
+        .await?
+        .success()
+    {
+        unmount("/var/lib/postgresql").await?;
+    }
+    let exists = tokio::fs::metadata(&db_dir).await.is_ok();
+    if !exists {
+        Command::new("cp")
+            .arg("-ra")
+            .arg("/var/lib/postgresql")
+            .arg(&db_dir)
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+    }
+    Command::new("chown")
+        .arg("-R")
+        .arg("postgres:postgres")
+        .arg(&db_dir)
+        .invoke(crate::ErrorKind::Database)
+        .await?;
+
+    let mut pg_paths = tokio::fs::read_dir("/usr/lib/postgresql").await?;
+    let mut pg_version = None;
+    while let Some(pg_path) = pg_paths.next_entry().await? {
+        let pg_path_version = pg_path
+            .file_name()
+            .to_str()
+            .map(|v| v.parse())
+            .transpose()?
+            .unwrap_or(0);
+        if pg_path_version > pg_version.unwrap_or(0) {
+            pg_version = Some(pg_path_version)
+        }
+    }
+    let pg_version = pg_version.ok_or_else(|| {
+        Error::new(
+            eyre!("could not determine postgresql version"),
+            crate::ErrorKind::Database,
+        )
+    })?;
+
+    crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
+
+    let pg_version_string = pg_version.to_string();
+    let pg_version_path = db_dir.join(&pg_version_string);
+    if tokio::fs::metadata(&pg_version_path).await.is_err() {
+        let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
+        let conf_dir_tmp = {
+            let mut tmp = conf_dir.clone();
+            tmp.set_extension("tmp");
+            tmp
+        };
+        if tokio::fs::metadata(&conf_dir).await.is_ok() {
+            Command::new("mv")
+                .arg(&conf_dir)
+                .arg(&conf_dir_tmp)
+                .invoke(ErrorKind::Filesystem)
+                .await?;
+        }
+        let mut old_version = pg_version;
+        while old_version > 13
+        /* oldest pg version included in startos */
+        {
+            old_version -= 1;
+            let old_datadir = db_dir.join(old_version.to_string());
+            if tokio::fs::metadata(&old_datadir).await.is_ok() {
+                Command::new("pg_upgradecluster")
+                    .arg(old_version.to_string())
+                    .arg("main")
+                    .invoke(crate::ErrorKind::Database)
+                    .await?;
+                break;
+            }
+        }
+        if tokio::fs::metadata(&conf_dir).await.is_ok() {
+            if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                tokio::fs::remove_dir_all(&conf_dir).await?;
+            }
+            Command::new("mv")
+                .arg(&conf_dir_tmp)
+                .arg(&conf_dir)
+                .invoke(ErrorKind::Filesystem)
+                .await?;
+        }
+    }
+
+    Command::new("systemctl")
+        .arg("start")
+        .arg(format!("postgresql@{pg_version}-main.service"))
+        .invoke(crate::ErrorKind::Database)
+        .await?;
+    if !exists {
+        Command::new("sudo")
+            .arg("-u")
+            .arg("postgres")
+            .arg("createuser")
+            .arg("root")
+            .invoke(crate::ErrorKind::Database)
+            .await?;
+        Command::new("sudo")
+            .arg("-u")
+            .arg("postgres")
+            .arg("createdb")
+            .arg("secrets")
+            .arg("-O")
+            .arg("root")
+            .invoke(crate::ErrorKind::Database)
+            .await?;
+    }
+
+    Ok(())
+}
+
+pub struct InitResult {
+    pub secret_store: Pool<Postgres>,
+    pub db: patch_db::PatchDb,
+}
+
+#[instrument(skip_all)]
+pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
+    tokio::fs::create_dir_all("/run/embassy")
+        .await
+        .with_ctx(|_| (crate::ErrorKind::Filesystem, "mkdir -p /run/embassy"))?;
+    if tokio::fs::metadata(LOCAL_AUTH_COOKIE_PATH).await.is_err() {
+        tokio::fs::write(
+            LOCAL_AUTH_COOKIE_PATH,
+            base64::encode(random::<[u8; 32]>()).as_bytes(),
+        )
+        .await
+        .with_ctx(|_| {
+            (
+                crate::ErrorKind::Filesystem,
+                format!("write {}", LOCAL_AUTH_COOKIE_PATH),
+            )
+        })?;
+        tokio::fs::set_permissions(LOCAL_AUTH_COOKIE_PATH, Permissions::from_mode(0o046)).await?;
+        Command::new("chown")
+            .arg("root:embassy")
+            .arg(LOCAL_AUTH_COOKIE_PATH)
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+    }
+
+    let secret_store = cfg.secret_store().await?;
+    tracing::info!("Opened Postgres");
+
+    crate::ssh::sync_keys_from_db(&secret_store, "/home/start9/.ssh/authorized_keys").await?;
+    tracing::info!("Synced SSH Keys");
+
+    let account = AccountInfo::load(&secret_store).await?;
+    let db = cfg.db(&account).await?;
+    tracing::info!("Opened PatchDB");
+    let peek = db.peek().await;
+    let mut server_info = peek.as_server_info().de()?;
+
+    // write to ca cert store
+    tokio::fs::write(
+        "/usr/local/share/ca-certificates/startos-root-ca.crt",
+        account.root_ca_cert.to_pem()?,
+    )
+    .await?;
+    Command::new("update-ca-certificates")
+        .invoke(crate::ErrorKind::OpenSsl)
+        .await?;
+
+    if let Some(wifi_interface) = &cfg.wifi_interface {
+        crate::net::wifi::synchronize_wpa_supplicant_conf(
+            &cfg.datadir().join("main"),
+            wifi_interface,
+            &server_info.last_wifi_region,
+        )
+        .await?;
+        tracing::info!("Synchronized WiFi");
+    }
+
+    let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
+        || &*server_info.version < &emver::Version::new(0, 3, 2, 0)
+        || (*ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
+
+    let song = if should_rebuild {
+        Some(NonDetachingJoinHandle::from(tokio::spawn(async {
+            loop {
+                BEP.play().await.unwrap();
+                BEP.play().await.unwrap();
+                tokio::time::sleep(Duration::from_secs(60)).await;
+            }
+        })))
+    } else {
+        None
+    };
+
+    let log_dir = cfg.datadir().join("main/logs");
+    if tokio::fs::metadata(&log_dir).await.is_err() {
+        tokio::fs::create_dir_all(&log_dir).await?;
+    }
+    let current_machine_id = tokio::fs::read_to_string("/etc/machine-id").await?;
+    let mut machine_ids = tokio::fs::read_dir(&log_dir).await?;
+    while let Some(machine_id) = machine_ids.next_entry().await? {
+        if machine_id.file_name().to_string_lossy().trim() != current_machine_id.trim() {
+            tokio::fs::remove_dir_all(machine_id.path()).await?;
+        }
+    }
+    crate::disk::mount::util::bind(&log_dir, "/var/log/journal", false).await?;
+    match Command::new("chattr")
+        .arg("-R")
+        .arg("+C")
+        .arg("/var/log/journal")
+        .invoke(ErrorKind::Filesystem)
+        .await
+    {
+        Ok(_) => Ok(()),
+        Err(e) if e.source.to_string().contains("Operation not supported") => Ok(()),
+        Err(e) => Err(e),
+    }?;
+    Command::new("systemctl")
+        .arg("restart")
+        .arg("systemd-journald")
+        .invoke(crate::ErrorKind::Journald)
+        .await?;
+    tracing::info!("Mounted Logs");
+
+    let tmp_dir = cfg.datadir().join("package-data/tmp");
+    if should_rebuild && tokio::fs::metadata(&tmp_dir).await.is_ok() {
+        tokio::fs::remove_dir_all(&tmp_dir).await?;
+    }
+    if tokio::fs::metadata(&tmp_dir).await.is_err() {
+        tokio::fs::create_dir_all(&tmp_dir).await?;
+    }
+    let tmp_var = cfg.datadir().join(format!("package-data/tmp/var"));
+    if tokio::fs::metadata(&tmp_var).await.is_ok() {
+        tokio::fs::remove_dir_all(&tmp_var).await?;
+    }
+    crate::disk::mount::util::bind(&tmp_var, "/var/tmp", false).await?;
+    let tmp_docker = cfg
+        .datadir()
+        .join(format!("package-data/tmp/{CONTAINER_TOOL}"));
+    let tmp_docker_exists = tokio::fs::metadata(&tmp_docker).await.is_ok();
+    if CONTAINER_TOOL == "docker" {
+        Command::new("systemctl")
+            .arg("stop")
+            .arg("docker")
+            .invoke(crate::ErrorKind::Docker)
+            .await?;
+    }
+    crate::disk::mount::util::bind(&tmp_docker, CONTAINER_DATADIR, false).await?;
+
+    if CONTAINER_TOOL == "docker" {
+        Command::new("systemctl")
+            .arg("reset-failed")
+            .arg("docker")
+            .invoke(crate::ErrorKind::Docker)
+            .await?;
+        Command::new("systemctl")
+            .arg("start")
+            .arg("docker")
+            .invoke(crate::ErrorKind::Docker)
+            .await?;
+    }
+    tracing::info!("Mounted Docker Data");
+
+    if should_rebuild || !tmp_docker_exists {
+        if CONTAINER_TOOL == "docker" {
+            tracing::info!("Creating Docker Network");
+            create_bridge_network("start9", "172.18.0.1/24", "br-start9").await?;
+            tracing::info!("Created Docker Network");
+        }
+
+        tracing::info!("Loading System Docker Images");
+        crate::install::load_images("/usr/lib/startos/system-images").await?;
+        tracing::info!("Loaded System Docker Images");
+
+        tracing::info!("Loading Package Docker Images");
+        crate::install::load_images(cfg.datadir().join(PKG_ARCHIVE_DIR)).await?;
+        tracing::info!("Loaded Package Docker Images");
+    }
+
+    if CONTAINER_TOOL == "podman" {
+        crate::util::docker::remove_container("netdummy", true).await?;
+        Command::new("podman")
+            .arg("run")
+            .arg("-d")
+            .arg("--rm")
+            .arg("--network=start9")
+            .arg("--name=netdummy")
+            .arg("start9/x_system/utils:latest")
+            .arg("sleep")
+            .arg("infinity")
+            .invoke(crate::ErrorKind::Docker)
+            .await?;
+    }
+
+    tracing::info!("Enabling Docker QEMU Emulation");
+    Command::new(CONTAINER_TOOL)
+        .arg("run")
+        .arg("--privileged")
+        .arg("--rm")
+        .arg("start9/x_system/binfmt")
+        .arg("--install")
+        .arg("all")
+        .invoke(crate::ErrorKind::Docker)
+        .await?;
+    tracing::info!("Enabled Docker QEMU Emulation");
+
+    if current_governor()
+        .await?
+        .map(|g| &g != &GOVERNOR_PERFORMANCE)
+        .unwrap_or(false)
+    {
+        tracing::info!("Setting CPU Governor to \"{}\"", GOVERNOR_PERFORMANCE);
+        if get_available_governors()
+            .await?
+            .contains(&GOVERNOR_PERFORMANCE)
+        {
+            set_governor(&GOVERNOR_PERFORMANCE).await?;
+            tracing::info!("Set CPU Governor");
+        } else {
+            tracing::warn!("CPU Governor \"{}\" Not Available", GOVERNOR_PERFORMANCE)
+        }
+    }
+
+    let mut time_not_synced = true;
+    let mut not_made_progress = 0u32;
+    for _ in 0..1800 {
+        if check_time_is_synchronized().await? {
+            time_not_synced = false;
+            break;
+        }
+        let t = SystemTime::now();
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        if t.elapsed()
+            .map(|t| t > Duration::from_secs_f64(1.1))
+            .unwrap_or(true)
+        {
+            not_made_progress = 0;
+        } else {
+            not_made_progress += 1;
+        }
+        if not_made_progress > 30 {
+            break;
+        }
+    }
+    if time_not_synced {
+        tracing::warn!("Timed out waiting for system time to synchronize");
+    } else {
+        tracing::info!("Syncronized system clock");
+    }
+
+    if server_info.zram {
+        crate::system::enable_zram().await?
+    }
+    server_info.ip_info = crate::net::dhcp::init_ips().await?;
+    server_info.status_info = ServerStatus {
+        updated: false,
+        update_progress: None,
+        backup_progress: None,
+        shutting_down: false,
+        restarting: false,
+    };
+
+    server_info.ntp_synced = if time_not_synced {
+        let db = db.clone();
+        tokio::spawn(async move {
+            while !check_time_is_synchronized().await.unwrap() {
+                tokio::time::sleep(Duration::from_secs(30)).await;
+            }
+            db.mutate(|v| v.as_server_info_mut().as_ntp_synced_mut().ser(&true))
+                .await
+                .unwrap()
+        });
+        false
+    } else {
+        true
+    };
+
+    db.mutate(|v| {
+        v.as_server_info_mut().ser(&server_info)?;
+        Ok(())
+    })
+    .await?;
+
+    crate::version::init(&db, &secret_store).await?;
+
+    db.mutate(|d| {
+        let model = d.de()?;
+        d.ser(&model)
+    })
+    .await?;
+
+    if should_rebuild {
+        match tokio::fs::remove_file(SYSTEM_REBUILD_PATH).await {
+            Ok(()) => Ok(()),
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+            Err(e) => Err(e),
+        }?;
+    }
+
+    drop(song);
+
+    tracing::info!("System initialized.");
+
+    Ok(InitResult { secret_store, db })
+}

core/startos/src/inspect.rs

@@ -0,0 +1,92 @@
+use std::path::PathBuf;
+
+use rpc_toolkit::command;
+
+use crate::s9pk::manifest::Manifest;
+use crate::s9pk::reader::S9pkReader;
+use crate::util::display_none;
+use crate::util::serde::{display_serializable, IoFormat};
+use crate::Error;
+
+#[command(subcommands(hash, manifest, license, icon, instructions, docker_images))]
+pub fn inspect() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(cli_only)]
+pub async fn hash(#[arg] path: PathBuf) -> Result<String, Error> {
+    Ok(S9pkReader::open(path, true)
+        .await?
+        .hash_str()
+        .unwrap()
+        .to_owned())
+}
+
+#[command(cli_only, display(display_serializable))]
+pub async fn manifest(
+    #[arg] path: PathBuf,
+    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+) -> Result<Manifest, Error> {
+    S9pkReader::open(path, !no_verify).await?.manifest().await
+}
+
+#[command(cli_only, display(display_none))]
+pub async fn license(
+    #[arg] path: PathBuf,
+    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
+) -> Result<(), Error> {
+    tokio::io::copy(
+        &mut S9pkReader::open(path, !no_verify).await?.license().await?,
+        &mut tokio::io::stdout(),
+    )
+    .await?;
+    Ok(())
+}
+
+#[command(cli_only, display(display_none))]
+pub async fn icon(
+    #[arg] path: PathBuf,
+    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
+) -> Result<(), Error> {
+    tokio::io::copy(
+        &mut S9pkReader::open(path, !no_verify).await?.icon().await?,
+        &mut tokio::io::stdout(),
+    )
+    .await?;
+    Ok(())
+}
+
+#[command(cli_only, display(display_none))]
+pub async fn instructions(
+    #[arg] path: PathBuf,
+    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
+) -> Result<(), Error> {
+    tokio::io::copy(
+        &mut S9pkReader::open(path, !no_verify)
+            .await?
+            .instructions()
+            .await?,
+        &mut tokio::io::stdout(),
+    )
+    .await?;
+    Ok(())
+}
+
+#[command(cli_only, display(display_none), rename = "docker-images")]
+pub async fn docker_images(
+    #[arg] path: PathBuf,
+    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
+) -> Result<(), Error> {
+    tokio::io::copy(
+        &mut S9pkReader::open(path, !no_verify)
+            .await?
+            .docker_images()
+            .await?,
+        &mut tokio::io::stdout(),
+    )
+    .await?;
+    Ok(())
+}

core/startos/src/install/cleanup.rs

@@ -0,0 +1,241 @@
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use models::OptionExt;
+use sqlx::{Executor, Postgres};
+use tracing::instrument;
+
+use super::PKG_ARCHIVE_DIR;
+use crate::context::RpcContext;
+use crate::db::model::{
+    CurrentDependencies, Database, PackageDataEntry, PackageDataEntryInstalled,
+    PackageDataEntryMatchModelRef,
+};
+use crate::error::ErrorCollection;
+use crate::prelude::*;
+use crate::s9pk::manifest::PackageId;
+use crate::util::{Apply, Version};
+use crate::volume::{asset_dir, script_dir};
+use crate::Error;
+
+#[instrument(skip_all)]
+pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Result<(), Error> {
+    let mut errors = ErrorCollection::new();
+    ctx.managers.remove(&(id.clone(), version.clone())).await;
+    // docker images start9/$APP_ID/*:$VERSION -q | xargs docker rmi
+    let images = crate::util::docker::images_for(id, version).await?;
+    errors.extend(
+        futures::future::join_all(images.into_iter().map(|sha| async {
+            let sha = sha; // move into future
+            crate::util::docker::remove_image(&sha).await
+        }))
+        .await,
+    );
+    let pkg_archive_dir = ctx
+        .datadir
+        .join(PKG_ARCHIVE_DIR)
+        .join(id)
+        .join(version.as_str());
+    if tokio::fs::metadata(&pkg_archive_dir).await.is_ok() {
+        tokio::fs::remove_dir_all(&pkg_archive_dir)
+            .await
+            .apply(|res| errors.handle(res));
+    }
+    let assets_path = asset_dir(&ctx.datadir, id, version);
+    if tokio::fs::metadata(&assets_path).await.is_ok() {
+        tokio::fs::remove_dir_all(&assets_path)
+            .await
+            .apply(|res| errors.handle(res));
+    }
+    let scripts_path = script_dir(&ctx.datadir, id, version);
+    if tokio::fs::metadata(&scripts_path).await.is_ok() {
+        tokio::fs::remove_dir_all(&scripts_path)
+            .await
+            .apply(|res| errors.handle(res));
+    }
+
+    errors.into_result()
+}
+
+#[instrument(skip_all)]
+pub async fn cleanup_failed(ctx: &RpcContext, id: &PackageId) -> Result<(), Error> {
+    if let Some(version) = match ctx
+        .db
+        .peek()
+        .await
+        .as_package_data()
+        .as_idx(id)
+        .or_not_found(id)?
+        .as_match()
+    {
+        PackageDataEntryMatchModelRef::Installing(m) => Some(m.as_manifest().as_version().de()?),
+        PackageDataEntryMatchModelRef::Restoring(m) => Some(m.as_manifest().as_version().de()?),
+        PackageDataEntryMatchModelRef::Updating(m) => {
+            let manifest_version = m.as_manifest().as_version().de()?;
+            let installed = m.as_installed().as_manifest().as_version().de()?;
+            if manifest_version != installed {
+                Some(manifest_version)
+            } else {
+                None // do not remove existing data
+            }
+        }
+        _ => {
+            tracing::warn!("{}: Nothing to clean up!", id);
+            None
+        }
+    } {
+        cleanup(ctx, id, &version).await?;
+    }
+
+    ctx.db
+        .mutate(|v| {
+            match v
+                .clone()
+                .into_package_data()
+                .into_idx(id)
+                .or_not_found(id)?
+                .as_match()
+            {
+                PackageDataEntryMatchModelRef::Installing(_)
+                | PackageDataEntryMatchModelRef::Restoring(_) => {
+                    v.as_package_data_mut().remove(id)?;
+                }
+                PackageDataEntryMatchModelRef::Updating(pde) => {
+                    v.as_package_data_mut()
+                        .as_idx_mut(id)
+                        .or_not_found(id)?
+                        .ser(&PackageDataEntry::Installed(PackageDataEntryInstalled {
+                            manifest: pde.as_installed().as_manifest().de()?,
+                            static_files: pde.as_static_files().de()?,
+                            installed: pde.as_installed().de()?,
+                        }))?;
+                }
+                _ => (),
+            }
+            Ok(())
+        })
+        .await
+}
+
+#[instrument(skip_all)]
+pub fn remove_from_current_dependents_lists(
+    db: &mut Model<Database>,
+    id: &PackageId,
+    current_dependencies: &CurrentDependencies,
+) -> Result<(), Error> {
+    for dep in current_dependencies.0.keys().chain(std::iter::once(id)) {
+        if let Some(current_dependents) = db
+            .as_package_data_mut()
+            .as_idx_mut(dep)
+            .and_then(|d| d.as_installed_mut())
+            .map(|i| i.as_current_dependents_mut())
+        {
+            current_dependents.remove(id)?;
+        }
+    }
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn uninstall<Ex>(ctx: &RpcContext, secrets: &mut Ex, id: &PackageId) -> Result<(), Error>
+where
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
+{
+    let db = ctx.db.peek().await;
+    let entry = db
+        .as_package_data()
+        .as_idx(id)
+        .or_not_found(id)?
+        .expect_as_removing()?;
+
+    let dependents_paths: Vec<PathBuf> = entry
+        .as_removing()
+        .as_current_dependents()
+        .keys()?
+        .into_iter()
+        .filter(|x| x != id)
+        .flat_map(|x| db.as_package_data().as_idx(&x))
+        .flat_map(|x| x.as_installed())
+        .flat_map(|x| x.as_manifest().as_volumes().de())
+        .flat_map(|x| x.values().cloned().collect::<Vec<_>>())
+        .flat_map(|x| x.pointer_path(&ctx.datadir))
+        .collect();
+
+    let volume_dir = ctx
+        .datadir
+        .join(crate::volume::PKG_VOLUME_DIR)
+        .join(&*entry.as_manifest().as_id().de()?);
+    let version = entry.as_removing().as_manifest().as_version().de()?;
+    tracing::debug!(
+        "Cleaning up {:?} except for {:?}",
+        volume_dir,
+        dependents_paths
+    );
+    cleanup(ctx, id, &version).await?;
+    cleanup_folder(volume_dir, Arc::new(dependents_paths)).await;
+    remove_network_keys(secrets, id).await?;
+
+    ctx.db
+        .mutate(|d| {
+            d.as_package_data_mut().remove(id)?;
+            remove_from_current_dependents_lists(
+                d,
+                id,
+                &entry.as_removing().as_current_dependencies().de()?,
+            )
+        })
+        .await
+}
+
+#[instrument(skip_all)]
+pub async fn remove_network_keys<Ex>(secrets: &mut Ex, id: &PackageId) -> Result<(), Error>
+where
+    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
+{
+    sqlx::query!("DELETE FROM network_keys WHERE package = $1", &*id)
+        .execute(&mut *secrets)
+        .await?;
+    sqlx::query!("DELETE FROM tor WHERE package = $1", &*id)
+        .execute(&mut *secrets)
+        .await?;
+    Ok(())
+}
+
+/// Needed to remove, without removing the folders that are mounted in the other docker containers
+pub fn cleanup_folder(
+    path: PathBuf,
+    dependents_volumes: Arc<Vec<PathBuf>>,
+) -> futures::future::BoxFuture<'static, ()> {
+    Box::pin(async move {
+        let meta_data = match tokio::fs::metadata(&path).await {
+            Ok(a) => a,
+            Err(_e) => {
+                return;
+            }
+        };
+        if !meta_data.is_dir() {
+            tracing::error!("is_not dir, remove {:?}", path);
+            let _ = tokio::fs::remove_file(&path).await;
+            return;
+        }
+        if !dependents_volumes
+            .iter()
+            .any(|v| v.starts_with(&path) || v == &path)
+        {
+            tracing::error!("No parents, remove {:?}", path);
+            let _ = tokio::fs::remove_dir_all(&path).await;
+            return;
+        }
+        let mut read_dir = match tokio::fs::read_dir(&path).await {
+            Ok(a) => a,
+            Err(_e) => {
+                return;
+            }
+        };
+        tracing::error!("Parents, recurse {:?}", path);
+        while let Some(entry) = read_dir.next_entry().await.ok().flatten() {
+            let entry_path = entry.path();
+            cleanup_folder(entry_path, dependents_volumes.clone()).await;
+        }
+    })
+}