Gateways, domains, and new service interface (#3001)

* add support for inbound proxies

* backend changes

* fix file type

* proxy -> tunnel, implement backend apis

* wip start-tunneld

* add domains and gateways, remove routers, fix docs links

* dont show hidden actions

* show and test dns

* edit instead of chnage acme and change gateway

* refactor: domains page

* refactor: gateways page

* domains and acme refactor

* certificate authorities

* refactor public/private gateways

* fix fe types

* domains mostly finished

* refactor: add file control to form service

* add ip util to sdk

* domains api + migration

* start service interface page, WIP

* different options for clearnet domains

* refactor: styles for interfaces page

* minor

* better placeholder for no addresses

* start sorting addresses

* best address logic

* comments

* fix unnecessary export

* MVP of service interface page

* domains preferred

* fix: address comments

* only translations left

* wip: start-tunnel & fix build

* forms for adding domain, rework things based on new ideas

* fix: dns testing

* public domain, max width, descriptions for dns

* nix StartOS domains, implement public and private domains at interface scope

* restart tor instead of reset

* better icon for restart tor

* dns

* fix sort functions for public and private domains

* with todos

* update types

* clean up tech debt, bump dependencies

* revert to ts-rs v9

* fix all types

* fix dns form

* add missing translations

* it builds

* fix: comments (#3009)

* fix: comments

* undo default

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* fix: refactor legacy components (#3010)

* fix: comments

* fix: refactor legacy components

* remove default again

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* more translations

* wip

* fix deadlock

* coukd work

* simple renaming

* placeholder for empty service interfaces table

* honor hidden form values

* remove logs

* reason instead of description

* fix dns

* misc fixes

* implement toggling gateways for service interface

* fix showing dns records

* move status column in service list

* remove unnecessary truthy check

* refactor: refactor forms components and remove legacy Taiga UI package (#3012)

* handle wh file uploads

* wip: debugging tor

* socks5 proxy working

* refactor: fix multiple comments (#3013)

* refactor: fix multiple comments

* styling changes, add documentation to sidebar

* translations for dns page

* refactor: subtle colors

* rearrange service page

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* fix file_stream and remove non-terminating test

* clean  up logs

* support for sccache

* fix gha sccache

* more marketplace translations

* install wizard clarity

* stub hostnameInfo in migration

* fix address info after setup, fix styling on SI page, new 040 release notes

* remove tor logs from os

* misc fixes

* reset tor still not functioning...

* update ts

* minor styling and wording

* chore: some fixes (#3015)

* fix gateway renames

* different handling for public domains

* styling fixes

* whole navbar should not be clickable on service show page

* timeout getState request

* remove links from changelog

* misc fixes from pairing

* use custom name for gateway in more places

* fix dns parsing

* closes #3003

* closes #2999

* chore: some fixes (#3017)

* small copy change

* revert hardcoded error for testing

* dont require port forward if gateway is public

* use old wan ip when not available

* fix .const hanging on undefined

* fix test

* fix doc test

* fix renames

* update deps

* allow specifying dependency metadata directly

* temporarily make dependencies not cliackable in marketplace listings

* fix socks bind

* fix test

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: waterplea <alexander@inkin.ru>

core/startos/src/service/effects/action.rs

@@ -1,9 +1,9 @@
 use std::collections::BTreeSet;
 
 use models::{ActionId, PackageId, ReplayId};
-use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
+use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
 
-use crate::action::{display_action_result, ActionInput, ActionResult};
+use crate::action::{ActionInput, ActionResult, display_action_result};
 use crate::db::model::package::{
     ActionMetadata, Task, TaskCondition, TaskEntry, TaskSeverity, TaskTrigger,
 };

core/startos/src/service/effects/callbacks.rs

@@ -264,7 +264,6 @@ impl CallbackHandler {
         }
     }
     pub async fn call(mut self, args: Vector<Value>) -> Result<(), Error> {
-        crate::dbg!(eyre!("callback fired: {}", self.handle.is_active()));
         if let Some(seed) = self.seed.upgrade() {
             seed.persistent_container
                 .callback(self.handle.take(), args)

core/startos/src/service/effects/control.rs

@@ -3,9 +3,9 @@ use std::str::FromStr;
 use clap::builder::ValueParserFactory;
 use models::{FromStrParser, PackageId};
 
+use crate::service::RebuildParams;
 use crate::service::effects::prelude::*;
 use crate::service::rpc::CallbackId;
-use crate::service::RebuildParams;
 use crate::status::MainStatus;
 
 pub async fn rebuild(context: EffectContext) -> Result<(), Error> {
@@ -21,21 +21,15 @@ pub async fn rebuild(context: EffectContext) -> Result<(), Error> {
     Ok(())
 }
 
-pub async fn restart(
-    context: EffectContext,
-    ProcedureId { procedure_id }: ProcedureId,
-) -> Result<(), Error> {
+pub async fn restart(context: EffectContext, EventId { event_id }: EventId) -> Result<(), Error> {
     let context = context.deref()?;
-    context.restart(procedure_id, false).await?;
+    context.restart(event_id, false).await?;
     Ok(())
 }
 
-pub async fn shutdown(
-    context: EffectContext,
-    ProcedureId { procedure_id }: ProcedureId,
-) -> Result<(), Error> {
+pub async fn shutdown(context: EffectContext, EventId { event_id }: EventId) -> Result<(), Error> {
     let context = context.deref()?;
-    context.stop(procedure_id, false).await?;
+    context.stop(event_id, false).await?;
     Ok(())
 }
 

core/startos/src/service/effects/dependency.rs

@@ -8,6 +8,7 @@ use imbl::OrdMap;
 use imbl_value::InternedString;
 use models::{FromStrParser, HealthCheckId, PackageId, ReplayId, VersionString, VolumeId};
 
+use crate::DATA_DIR;
 use crate::db::model::package::{
     CurrentDependencies, CurrentDependencyInfo, CurrentDependencyKind, ManifestPreference,
     TaskEntry,
@@ -19,7 +20,6 @@ use crate::disk::mount::util::{is_mountpoint, unmount};
 use crate::service::effects::prelude::*;
 use crate::status::health_check::NamedHealthCheckResult;
 use crate::volume::data_dir;
-use crate::DATA_DIR;
 
 #[derive(Debug, Clone, Serialize, Deserialize, TS)]
 #[ts(export)]

core/startos/src/service/effects/health.rs

@@ -1,8 +1,8 @@
 use models::HealthCheckId;
 
 use crate::service::effects::prelude::*;
-use crate::status::health_check::NamedHealthCheckResult;
 use crate::status::MainStatus;
+use crate::status::health_check::NamedHealthCheckResult;
 
 #[derive(Debug, Clone, Serialize, Deserialize, TS)]
 #[serde(rename_all = "camelCase")]
@@ -31,8 +31,7 @@ pub async fn set_health(
                 .as_status_mut()
                 .mutate(|main| {
                     match main {
-                        MainStatus::Running { ref mut health, .. }
-                        | MainStatus::Starting { ref mut health } => {
+                        MainStatus::Running { health, .. } | MainStatus::Starting { health } => {
                             health.insert(id, result);
                         }
                         _ => (),

core/startos/src/service/effects/mod.rs

@@ -1,11 +1,11 @@
 use std::net::Ipv4Addr;
 
-use rpc_toolkit::{from_fn, from_fn_async, from_fn_blocking, Context, HandlerExt, ParentHandler};
+use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn, from_fn_async, from_fn_blocking};
 
 use crate::prelude::*;
 use crate::service::cli::ContainerCliContext;
 use crate::service::effects::context::EffectContext;
-use crate::{echo, HOST_IP};
+use crate::{HOST_IP, echo};
 
 mod action;
 pub mod callbacks;

core/startos/src/service/effects/net/bind.rs

@@ -89,5 +89,6 @@ pub async fn get_service_port_forward(
         .de()?
         .get(&internal_port)
         .or_not_found(lazy_format!("binding for port {internal_port}"))?
-        .net)
+        .net
+        .clone())
 }

core/startos/src/service/effects/net/ssl.rs

@@ -6,11 +6,11 @@ use ipnet::IpNet;
 use itertools::Itertools;
 use openssl::pkey::{PKey, Private};
 
+use crate::HOST_IP;
 use crate::service::effects::callbacks::CallbackHandler;
 use crate::service::effects::prelude::*;
 use crate::service::rpc::CallbackId;
 use crate::util::serde::Pem;
-use crate::HOST_IP;
 
 #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, TS, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
@@ -59,7 +59,8 @@ pub async fn get_ssl_certificate(
                         .de()?
                         .iter()
                         .map(InternedString::from_display)
-                        .chain(m.as_domains().keys()?)
+                        .chain(m.as_public_domains().keys()?)
+                        .chain(m.as_private_domains().de()?)
                         .chain(
                             m.as_hostname_info()
                                 .de()?
@@ -90,7 +91,7 @@ pub async fn get_ssl_certificate(
                         .as_public()
                         .as_server_info()
                         .as_network()
-                        .as_network_interfaces()
+                        .as_gateways()
                         .as_entries()?
                         .into_iter()
                         .flat_map(|(_, net)| net.as_ip_info().transpose_ref())
@@ -184,7 +185,8 @@ pub async fn get_ssl_key(
                         .de()?
                         .iter()
                         .map(InternedString::from_display)
-                        .chain(m.as_domains().keys()?)
+                        .chain(m.as_public_domains().keys()?)
+                        .chain(m.as_private_domains().de()?)
                         .chain(
                             m.as_hostname_info()
                                 .de()?

core/startos/src/service/effects/prelude.rs

@@ -9,8 +9,8 @@ pub(super) use crate::service::effects::context::EffectContext;
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Parser, TS)]
 #[serde(rename_all = "camelCase")]
 #[ts(export)]
-pub struct ProcedureId {
+pub struct EventId {
     #[serde(default)]
     #[arg(default_value_t, long)]
-    pub procedure_id: Guid,
+    pub event_id: Guid,
 }

core/startos/src/service/effects/subcontainer/mod.rs

@@ -11,14 +11,14 @@ use crate::service::effects::prelude::*;
 use crate::service::persistent_container::Subcontainer;
 use crate::util::Invoke;
 
-#[cfg(feature = "container-runtime")]
+#[cfg(feature = "cli-container")]
 mod sync;
 
-#[cfg(not(feature = "container-runtime"))]
+#[cfg(not(feature = "cli-container"))]
 mod sync_dummy;
 
 pub use sync::*;
-#[cfg(not(feature = "container-runtime"))]
+#[cfg(not(feature = "cli-container"))]
 use sync_dummy as sync;
 
 #[derive(Debug, Deserialize, Serialize, Parser, TS)]
@@ -61,7 +61,9 @@ pub async fn destroy_subcontainer_fs(
         }
         overlay.overlay.unmount(true).await?;
     } else {
-        tracing::warn!("Could not find a subcontainer fs to destroy; assumming that it already is destroyed and will be skipping");
+        tracing::warn!(
+            "Could not find a subcontainer fs to destroy; assumming that it already is destroyed and will be skipping"
+        );
     }
     Ok(())
 }

core/startos/src/service/effects/subcontainer/sync.rs

@@ -1,5 +1,5 @@
 use std::collections::BTreeMap;
-use std::ffi::{c_int, OsStr, OsString};
+use std::ffi::{OsStr, OsString, c_int};
 use std::fs::File;
 use std::io::{IsTerminal, Read};
 use std::os::unix::process::{CommandExt, ExitStatusExt};
@@ -13,10 +13,10 @@ use signal_hook::consts::signal::*;
 use termion::raw::IntoRawMode;
 use tokio::sync::oneshot;
 
-use crate::service::effects::prelude::*;
-use crate::service::effects::ContainerCliContext;
-use crate::util::io::TermSize;
 use crate::CAP_1_KiB;
+use crate::service::effects::ContainerCliContext;
+use crate::service::effects::prelude::*;
+use crate::util::io::TermSize;
 
 const FWD_SIGNALS: &[c_int] = &[
     SIGABRT, SIGALRM, SIGCONT, SIGHUP, SIGINT, SIGIO, SIGPIPE, SIGPROF, SIGQUIT, SIGTERM, SIGTRAP,
@@ -284,7 +284,7 @@ pub fn launch(
     if tty {
         use pty_process::blocking as pty_process;
         let (pty, pts) = pty_process::open().with_kind(ErrorKind::Filesystem)?;
-        let mut cmd = pty_process::Command::new("/usr/bin/start-cli");
+        let mut cmd = pty_process::Command::new("/usr/bin/start-container");
         cmd = cmd.arg("subcontainer").arg("launch-init");
         if let Some(env) = env {
             cmd = cmd.arg("--env").arg(env);
@@ -339,7 +339,7 @@ pub fn launch(
             ))
         }
     } else {
-        let mut cmd = StdCommand::new("/usr/bin/start-cli");
+        let mut cmd = StdCommand::new("/usr/bin/start-container");
         cmd.arg("subcontainer").arg("launch-init");
         if let Some(env) = env {
             cmd.arg("--env").arg(env);
@@ -534,7 +534,7 @@ pub fn exec(
     if tty {
         use pty_process::blocking as pty_process;
         let (pty, pts) = pty_process::open().with_kind(ErrorKind::Filesystem)?;
-        let mut cmd = pty_process::Command::new("/usr/bin/start-cli");
+        let mut cmd = pty_process::Command::new("/usr/bin/start-container");
         cmd = cmd.arg("subcontainer").arg("exec-command");
         if let Some(env) = env {
             cmd = cmd.arg("--env").arg(env);
@@ -589,7 +589,7 @@ pub fn exec(
             ))
         }
     } else {
-        let mut cmd = StdCommand::new("/usr/bin/start-cli");
+        let mut cmd = StdCommand::new("/usr/bin/start-container");
         cmd.arg("subcontainer").arg("exec-command");
         if let Some(env) = env {
             cmd.arg("--env").arg(env);

core/startos/src/service/effects/subcontainer/sync_dummy.rs

@@ -1,5 +1,5 @@
-use crate::service::effects::prelude::*;
 use crate::service::effects::ContainerCliContext;
+use crate::service::effects::prelude::*;
 
 pub fn launch(_: ContainerCliContext) -> Result<(), Error> {
     Err(Error::new(

core/startos/src/service/effects/version.rs

@@ -1,9 +1,9 @@
 use std::path::Path;
 
+use crate::DATA_DIR;
 use crate::service::effects::prelude::*;
 use crate::util::io::{delete_file, maybe_read_file_to_string, write_file_atomic};
 use crate::volume::PKG_VOLUME_DIR;
-use crate::DATA_DIR;
 
 #[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)]
 #[serde(rename_all = "camelCase")]