Refactor/project structure (#3085)

* refactor project structure

* environment-based default registry

* fix tests

* update build container

* use docker platform for iso build emulation

* simplify compat

* Fix docker platform spec in run-compat.sh

* handle riscv compat

* fix bug with dep error exists attr

* undo removal of sorting

* use qemu for iso stage

---------

Co-authored-by: Mariusz Kogen <k0gen@pm.me>
Co-authored-by: Matt Hill <mattnine@protonmail.com>

core/Cargo.toml

@@ -1,3 +1,290 @@
-[workspace]
+[package]
+authors = ["Aiden McClelland <me@drbonez.dev>"]
+description = "The core of StartOS"
+documentation = "https://docs.rs/start-os"
+edition = "2024"
+keywords = [
+  "bitcoin",
+  "full-node",
+  "lightning",
+  "privacy",
+  "raspberry-pi",
+  "self-hosted",
+]
+license = "MIT"
+name = "start-os"
+readme = "README.md"
+repository = "https://github.com/Start9Labs/start-os"
+version = "0.4.0-alpha.16" # VERSION_BUMP
 
-members = ["startos"]
+[lib]
+name = "startos"
+path = "src/lib.rs"
+
+[[bin]]
+name = "startbox"
+path = "src/main/startbox.rs"
+
+[[bin]]
+name = "start-cli"
+path = "src/main/start-cli.rs"
+
+[[bin]]
+name = "start-container"
+path = "src/main/start-container.rs"
+
+[[bin]]
+name = "registrybox"
+path = "src/main/registrybox.rs"
+
+[[bin]]
+name = "tunnelbox"
+path = "src/main/tunnelbox.rs"
+
+[features]
+arti = [
+  "arti-client",
+  "safelog",
+  "tor-cell",
+  "tor-hscrypto",
+  "tor-hsservice",
+  "tor-keymgr",
+  "tor-llcrypto",
+  "tor-proto",
+  "tor-rtcompat",
+]
+beta = []
+console = ["console-subscriber", "tokio/tracing"]
+default = []
+dev = []
+test = []
+unstable = ["backtrace-on-stack-overflow"]
+
+[dependencies]
+aes = { version = "0.7.5", features = ["ctr"] }
+arti-client = { version = "0.33", features = [
+  "compression",
+  "ephemeral-keystore",
+  "experimental-api",
+  "onion-service-client",
+  "onion-service-service",
+  "rustls",
+  "static",
+  "tokio",
+], default-features = false, git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+async-acme = { version = "0.6.0", git = "https://github.com/dr-bonez/async-acme.git", features = [
+  "use_rustls",
+  "use_tokio",
+] }
+async-compression = { version = "0.4.32", features = [
+  "brotli",
+  "gzip",
+  "tokio",
+  "zstd",
+] }
+async-stream = "0.3.5"
+async-trait = "0.1.74"
+axum = { version = "0.8.4", features = ["ws", "http2"] }
+backtrace-on-stack-overflow = { version = "0.3.0", optional = true }
+base32 = "0.5.0"
+base64 = "0.22.1"
+base64ct = "1.6.0"
+basic-cookies = "0.1.4"
+blake3 = { version = "1.5.0", features = ["mmap", "rayon"] }
+bytes = "1"
+chrono = { version = "0.4.31", features = ["serde"] }
+clap = { version = "4.4.12", features = ["string"] }
+color-eyre = "0.6.2"
+console = "0.16.2"
+console-subscriber = { version = "0.5.0", optional = true }
+const_format = "0.2.34"
+cookie = "0.18.0"
+cookie_store = "0.22.0"
+curve25519-dalek = "4.1.3"
+der = { version = "0.7.9", features = ["derive", "pem"] }
+digest = "0.10.7"
+divrem = "1.0.0"
+dns-lookup = "3.0.1"
+ed25519 = { version = "2.2.3", features = ["alloc", "pem", "pkcs8"] }
+ed25519-dalek = { version = "2.2.0", features = [
+  "digest",
+  "hazmat",
+  "pkcs8",
+  "rand_core",
+  "serde",
+  "zeroize",
+] }
+ed25519-dalek-v1 = { package = "ed25519-dalek", version = "1" }
+exver = { version = "0.2.0", git = "https://github.com/Start9Labs/exver-rs.git", features = [
+  "serde",
+] }
+fd-lock-rs = "0.1.4"
+form_urlencoded = "1.2.1"
+futures = "0.3.28"
+gpt = "4.1.0"
+hex = "0.4.3"
+hickory-server = { version = "0.25.2", features = ["resolver"] }
+hmac = "0.12.1"
+http = "1.0.0"
+http-body-util = "0.1"
+hyper = { version = "1.5", features = ["http1", "http2", "server"] }
+hyper-util = { version = "0.1.10", features = [
+  "http1",
+  "http2",
+  "server",
+  "server-auto",
+  "server-graceful",
+  "service",
+  "tokio",
+] }
+id-pool = { version = "0.2.2", default-features = false, features = [
+  "serde",
+  "u16",
+] }
+iddqd = "0.3.14"
+imbl = { version = "6", features = ["serde", "small-chunks"] }
+imbl-value = { version = "0.4.3", features = ["ts-rs"] }
+include_dir = { version = "0.7.3", features = ["metadata"] }
+indexmap = { version = "2.0.2", features = ["serde"] }
+indicatif = { version = "0.18.3", features = ["tokio"] }
+inotify = "0.11.0"
+integer-encoding = { version = "4.0.0", features = ["tokio_async"] }
+ipnet = { version = "2.8.0", features = ["serde"] }
+isocountry = "0.3.2"
+itertools = "0.14.0"
+jaq-core = "0.10.1"
+jaq-std = "0.10.0"
+josekit = "0.10.3"
+jsonpath_lib = { git = "https://github.com/Start9Labs/jsonpath.git" }
+lazy_async_pool = "0.3.3"
+lazy_format = "2.0"
+lazy_static = "1.4.0"
+lettre = { version = "0.11.18", default-features = false, features = [
+  "aws-lc-rs",
+  "builder",
+  "hostname",
+  "pool",
+  "rustls-platform-verifier",
+  "smtp-transport",
+  "tokio1-rustls",
+] }
+libc = "0.2.149"
+log = "0.4.20"
+mbrman = "0.6.0"
+miette = { version = "7.6.0", features = ["fancy"] }
+mio = "1"
+new_mime_guess = "4"
+nix = { version = "0.30.1", features = [
+  "fs",
+  "mount",
+  "net",
+  "process",
+  "sched",
+  "signal",
+  "user",
+] }
+nom = "8.0.0"
+num = "0.4.1"
+num_cpus = "1.16.0"
+num_enum = "0.7.0"
+once_cell = "1.19.0"
+openssh-keys = "0.6.2"
+openssl = { version = "0.10.57", features = ["vendored"] }
+p256 = { version = "0.13.2", features = ["pem"] }
+patch-db = { version = "*", path = "../patch-db/patch-db", features = [
+  "trace",
+] }
+pbkdf2 = "0.12.2"
+pin-project = "1.1.3"
+pkcs8 = { version = "0.10.2", features = ["std"] }
+prettytable-rs = "0.10.0"
+proptest = "1.3.1"
+proptest-derive = "0.7.0"
+qrcode = "0.14.1"
+r3bl_tui = "0.7.6"
+rand = "0.9.2"
+regex = "1.10.2"
+reqwest = { version = "0.12.25", features = [
+  "json",
+  "socks",
+  "stream",
+  "http2",
+] }
+reqwest_cookie_store = "0.9.0"
+rpassword = "7.2.0"
+rust-argon2 = "3.0.0"
+rpc-toolkit = { git = "https://github.com/Start9Labs/rpc-toolkit.git" }
+safelog = { version = "0.4.8", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+semver = { version = "1.0.20", features = ["serde"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
+serde_cbor = { package = "ciborium", version = "0.2.1" }
+serde_json = "1.0"
+serde_toml = { package = "toml", version = "0.9.9+spec-1.0.0" }
+serde_yaml = { package = "serde_yml", version = "0.0.12" }
+sha-crypt = "0.5.0"
+sha2 = "0.10.2"
+signal-hook = "0.3.17"
+socket2 = { version = "0.6.0", features = ["all"] }
+socks5-impl = { version = "0.7.2", features = ["client", "server"] }
+sqlx = { version = "0.8.6", features = [
+  "postgres",
+  "runtime-tokio-rustls",
+], default-features = false }
+sscanf = "0.4.1"
+ssh-key = { version = "0.6.2", features = ["ed25519"] }
+tar = "0.4.40"
+termion = "4.0.5"
+textwrap = "0.16.1"
+thiserror = "2.0.12"
+tokio = { version = "1.38.1", features = ["full"] }
+tokio-rustls = "0.26.4"
+tokio-stream = { version = "0.1.14", features = ["io-util", "net", "sync"] }
+tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
+tokio-tungstenite = { version = "0.26.2", features = ["native-tls", "url"] }
+tokio-util = { version = "0.7.9", features = ["io"] }
+tor-cell = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+tor-hscrypto = { version = "0.33", features = [
+  "full",
+], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+tor-hsservice = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+tor-keymgr = { version = "0.33", features = [
+  "ephemeral-keystore",
+], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+tor-llcrypto = { version = "0.33", features = [
+  "full",
+], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+tor-proto = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+tor-rtcompat = { version = "0.33", features = [
+  "rustls",
+  "tokio",
+], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
+torut = "0.2.1"
+tower-service = "0.3.3"
+tracing = "0.1.39"
+tracing-error = "0.2.0"
+tracing-journald = "0.3.0"
+tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
+ts-rs = "9.0.1"
+typed-builder = "0.23.2"
+url = { version = "2.4.1", features = ["serde"] }
+uuid = { version = "1.4.1", features = ["v4"] }
+visit-rs = "0.1.1"
+x25519-dalek = { version = "2.0.1", features = ["static_secrets"] }
+zbus = "5.1.1"
+hashing-serializer = "0.1.1"
+
+[target.'cfg(target_os = "linux")'.dependencies]
+procfs = "0.18.0"
+pty-process = "0.5.1"
+
+[profile.test]
+opt-level = 3
+
+[profile.dev]
+opt-level = 3
+
+[profile.dev.package.backtrace]
+opt-level = 3
+
+[profile.dev.package.sqlx-macros]
+opt-level = 3

core/build/build-cli.sh

@@ -60,7 +60,7 @@ if [ -z "${TARGET:-}" ]; then
   fi
 fi
 
-cd ..
+cd ../..
 FEATURES="$(echo "${ENVIRONMENT:-}" | sed 's/-/,/g')"
 RUSTFLAGS=""
 if [[ "${ENVIRONMENT:-}" =~ (^|-)console($|-) ]]; then

core/build/build-registrybox.sh

@@ -30,7 +30,7 @@ if [ "$ARCH" = "riscv64" ]; then
   RUST_ARCH="riscv64gc"
 fi
 
-cd ..
+cd ../..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 

core/build/build-start-container.sh

@@ -30,7 +30,7 @@ if [ "$ARCH" = "riscv64" ]; then
   RUST_ARCH="riscv64gc"
 fi
 
-cd ..
+cd ../..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 

core/build/build-startbox.sh

@@ -30,7 +30,7 @@ if [ "$ARCH" = "riscv64" ]; then
   RUST_ARCH="riscv64gc"
 fi
 
-cd ..
+cd ../..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 

core/build/build-ts.sh

@@ -30,7 +30,7 @@ if [ "$ARCH" = "riscv64" ]; then
 	RUST_ARCH="riscv64gc"
 fi
 
-cd ..
+cd ../..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 if [[ "${ENVIRONMENT}" =~ (^|-)console($|-) ]]; then
@@ -39,6 +39,6 @@ fi
 echo "FEATURES=\"$FEATURES\""
 echo "RUSTFLAGS=\"$RUSTFLAGS\""
 rust-zig-builder cargo test --manifest-path=./core/Cargo.toml $BUILD_FLAGS --features test,$FEATURES --locked 'export_bindings_'
-if [ "$(ls -nd "core/startos/bindings" | awk '{ print $3 }')" != "$UID" ]; then
-  rust-zig-builder sh -c "chown -R $UID:$UID core/target && chown -R $UID:$UID core/startos/bindings && chown -R $UID:$UID  /usr/local/cargo"
+if [ "$(ls -nd "core/bindings" | awk '{ print $3 }')" != "$UID" ]; then
+  rust-zig-builder sh -c "chown -R $UID:$UID core/target && chown -R $UID:$UID core/bindings && chown -R $UID:$UID  /usr/local/cargo"
 fi

core/build/build-tunnelbox.sh

@@ -30,7 +30,7 @@ if [ "$ARCH" = "riscv64" ]; then
   RUST_ARCH="riscv64gc"
 fi
 
-cd ..
+cd ../..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 

core/run-tests.sh

@@ -2,7 +2,7 @@
 
 cd "$(dirname "${BASH_SOURCE[0]}")"
 
-source ./builder-alias.sh
+source ./build/builder-alias.sh
 
 set -ea
 shopt -s expand_aliases

core/src/lib.rs

@@ -3,8 +3,6 @@ use const_format::formatcp;
 pub const DATA_DIR: &str = "/media/startos/data";
 pub const MAIN_DATA: &str = formatcp!("{DATA_DIR}/main");
 pub const PACKAGE_DATA: &str = formatcp!("{DATA_DIR}/package-data");
-pub const DEFAULT_REGISTRY: &str = "https://registry.start9.com";
-// pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com";
 pub const HOST_IP: [u8; 4] = [10, 0, 3, 1];
 pub use std::env::consts::ARCH;
 lazy_static::lazy_static! {

core/src/main/startbox.rs

@@ -3,23 +3,31 @@ use startos::bins::MultiExecutable;
 fn main() {
     startos::net::static_server::UI_CELL
         .set(include_dir::include_dir!(
-            "$CARGO_MANIFEST_DIR/../../web/dist/static/ui"
+            "$CARGO_MANIFEST_DIR/../web/dist/static/ui"
         ))
         .ok();
     startos::net::static_server::SETUP_WIZARD_CELL
         .set(include_dir::include_dir!(
-            "$CARGO_MANIFEST_DIR/../../web/dist/static/setup-wizard"
+            "$CARGO_MANIFEST_DIR/../web/dist/static/setup-wizard"
         ))
         .ok();
     startos::net::static_server::INSTALL_WIZARD_CELL
         .set(include_dir::include_dir!(
-            "$CARGO_MANIFEST_DIR/../../web/dist/static/install-wizard"
+            "$CARGO_MANIFEST_DIR/../web/dist/static/install-wizard"
         ))
         .ok();
+    #[cfg(not(feature = "beta"))]
     startos::db::model::public::DB_UI_SEED_CELL
         .set(include_str!(concat!(
             env!("CARGO_MANIFEST_DIR"),
-            "/../../web/patchdb-ui-seed.json"
+            "/../web/patchdb-ui-seed.json"
+        )))
+        .ok();
+    #[cfg(feature = "beta")]
+    startos::db::model::public::DB_UI_SEED_CELL
+        .set(include_str!(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/../web/patchdb-ui-seed.beta.json"
         )))
         .ok();
     MultiExecutable::default()

core/src/main/tunnelbox.rs

@@ -3,7 +3,7 @@ use startos::bins::MultiExecutable;
 fn main() {
     startos::tunnel::context::TUNNEL_UI_CELL
         .set(include_dir::include_dir!(
-            "$CARGO_MANIFEST_DIR/../../web/dist/static/start-tunnel"
+            "$CARGO_MANIFEST_DIR/../web/dist/static/start-tunnel"
         ))
         .ok();
     MultiExecutable::default()