Gateways, domains, and new service interface (#3001)

* add support for inbound proxies

* backend changes

* fix file type

* proxy -> tunnel, implement backend apis

* wip start-tunneld

* add domains and gateways, remove routers, fix docs links

* dont show hidden actions

* show and test dns

* edit instead of chnage acme and change gateway

* refactor: domains page

* refactor: gateways page

* domains and acme refactor

* certificate authorities

* refactor public/private gateways

* fix fe types

* domains mostly finished

* refactor: add file control to form service

* add ip util to sdk

* domains api + migration

* start service interface page, WIP

* different options for clearnet domains

* refactor: styles for interfaces page

* minor

* better placeholder for no addresses

* start sorting addresses

* best address logic

* comments

* fix unnecessary export

* MVP of service interface page

* domains preferred

* fix: address comments

* only translations left

* wip: start-tunnel & fix build

* forms for adding domain, rework things based on new ideas

* fix: dns testing

* public domain, max width, descriptions for dns

* nix StartOS domains, implement public and private domains at interface scope

* restart tor instead of reset

* better icon for restart tor

* dns

* fix sort functions for public and private domains

* with todos

* update types

* clean up tech debt, bump dependencies

* revert to ts-rs v9

* fix all types

* fix dns form

* add missing translations

* it builds

* fix: comments (#3009)

* fix: comments

* undo default

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* fix: refactor legacy components (#3010)

* fix: comments

* fix: refactor legacy components

* remove default again

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* more translations

* wip

* fix deadlock

* coukd work

* simple renaming

* placeholder for empty service interfaces table

* honor hidden form values

* remove logs

* reason instead of description

* fix dns

* misc fixes

* implement toggling gateways for service interface

* fix showing dns records

* move status column in service list

* remove unnecessary truthy check

* refactor: refactor forms components and remove legacy Taiga UI package (#3012)

* handle wh file uploads

* wip: debugging tor

* socks5 proxy working

* refactor: fix multiple comments (#3013)

* refactor: fix multiple comments

* styling changes, add documentation to sidebar

* translations for dns page

* refactor: subtle colors

* rearrange service page

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* fix file_stream and remove non-terminating test

* clean  up logs

* support for sccache

* fix gha sccache

* more marketplace translations

* install wizard clarity

* stub hostnameInfo in migration

* fix address info after setup, fix styling on SI page, new 040 release notes

* remove tor logs from os

* misc fixes

* reset tor still not functioning...

* update ts

* minor styling and wording

* chore: some fixes (#3015)

* fix gateway renames

* different handling for public domains

* styling fixes

* whole navbar should not be clickable on service show page

* timeout getState request

* remove links from changelog

* misc fixes from pairing

* use custom name for gateway in more places

* fix dns parsing

* closes #3003

* closes #2999

* chore: some fixes (#3017)

* small copy change

* revert hardcoded error for testing

* dont require port forward if gateway is public

* use old wan ip when not available

* fix .const hanging on undefined

* fix test

* fix doc test

* fix renames

* update deps

* allow specifying dependency metadata directly

* temporarily make dependencies not cliackable in marketplace listings

* fix socks bind

* fix test

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: waterplea <alexander@inkin.ru>
This commit is contained in:
Matt Hill
2025-09-09 21:43:51 -06:00
committed by GitHub
parent 1cc9a1a30b
commit add01ebc68
537 changed files with 19940 additions and 20551 deletions

View File

@@ -115,7 +115,7 @@ pub fn update_tasks(
}
pub(super) struct RunAction {
id: ActionId,
action_id: ActionId,
input: Value,
}
impl Handler<RunAction> for ServiceActor {
@@ -127,7 +127,7 @@ impl Handler<RunAction> for ServiceActor {
&mut self,
id: Guid,
RunAction {
id: ref action_id,
ref action_id,
input,
}: RunAction,
jobs: &BackgroundJobQueue,
@@ -145,7 +145,7 @@ impl Handler<RunAction> for ServiceActor {
.into_idx(package_id)
.or_not_found(package_id)?
.into_actions()
.into_idx(&action_id)
.into_idx(action_id)
.or_not_found(lazy_format!("{package_id} action {action_id}"))?
.de()?;
if matches!(&action.visibility, ActionVisibility::Disabled(_)) {
@@ -226,14 +226,6 @@ impl Service {
action_id: ActionId,
input: Value,
) -> Result<Option<ActionResult>, Error> {
self.actor
.send(
id,
RunAction {
id: action_id,
input,
},
)
.await?
self.actor.send(id, RunAction { action_id, input }).await?
}
}

View File

@@ -5,7 +5,7 @@ use clap::Parser;
use imbl_value::Value;
use once_cell::sync::OnceCell;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{call_remote_socket, yajrc, CallRemote, Context, Empty};
use rpc_toolkit::{CallRemote, Context, Empty, call_remote_socket, yajrc};
use tokio::runtime::Runtime;
use crate::lxc::HOST_RPC_SERVER_SOCKET;

View File

@@ -1,9 +1,9 @@
use std::collections::BTreeSet;
use models::{ActionId, PackageId, ReplayId};
use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
use crate::action::{display_action_result, ActionInput, ActionResult};
use crate::action::{ActionInput, ActionResult, display_action_result};
use crate::db::model::package::{
ActionMetadata, Task, TaskCondition, TaskEntry, TaskSeverity, TaskTrigger,
};

View File

@@ -264,7 +264,6 @@ impl CallbackHandler {
}
}
pub async fn call(mut self, args: Vector<Value>) -> Result<(), Error> {
crate::dbg!(eyre!("callback fired: {}", self.handle.is_active()));
if let Some(seed) = self.seed.upgrade() {
seed.persistent_container
.callback(self.handle.take(), args)

View File

@@ -3,9 +3,9 @@ use std::str::FromStr;
use clap::builder::ValueParserFactory;
use models::{FromStrParser, PackageId};
use crate::service::RebuildParams;
use crate::service::effects::prelude::*;
use crate::service::rpc::CallbackId;
use crate::service::RebuildParams;
use crate::status::MainStatus;
pub async fn rebuild(context: EffectContext) -> Result<(), Error> {
@@ -21,21 +21,15 @@ pub async fn rebuild(context: EffectContext) -> Result<(), Error> {
Ok(())
}
pub async fn restart(
context: EffectContext,
ProcedureId { procedure_id }: ProcedureId,
) -> Result<(), Error> {
pub async fn restart(context: EffectContext, EventId { event_id }: EventId) -> Result<(), Error> {
let context = context.deref()?;
context.restart(procedure_id, false).await?;
context.restart(event_id, false).await?;
Ok(())
}
pub async fn shutdown(
context: EffectContext,
ProcedureId { procedure_id }: ProcedureId,
) -> Result<(), Error> {
pub async fn shutdown(context: EffectContext, EventId { event_id }: EventId) -> Result<(), Error> {
let context = context.deref()?;
context.stop(procedure_id, false).await?;
context.stop(event_id, false).await?;
Ok(())
}

View File

@@ -8,6 +8,7 @@ use imbl::OrdMap;
use imbl_value::InternedString;
use models::{FromStrParser, HealthCheckId, PackageId, ReplayId, VersionString, VolumeId};
use crate::DATA_DIR;
use crate::db::model::package::{
CurrentDependencies, CurrentDependencyInfo, CurrentDependencyKind, ManifestPreference,
TaskEntry,
@@ -19,7 +20,6 @@ use crate::disk::mount::util::{is_mountpoint, unmount};
use crate::service::effects::prelude::*;
use crate::status::health_check::NamedHealthCheckResult;
use crate::volume::data_dir;
use crate::DATA_DIR;
#[derive(Debug, Clone, Serialize, Deserialize, TS)]
#[ts(export)]

View File

@@ -1,8 +1,8 @@
use models::HealthCheckId;
use crate::service::effects::prelude::*;
use crate::status::health_check::NamedHealthCheckResult;
use crate::status::MainStatus;
use crate::status::health_check::NamedHealthCheckResult;
#[derive(Debug, Clone, Serialize, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
@@ -31,8 +31,7 @@ pub async fn set_health(
.as_status_mut()
.mutate(|main| {
match main {
MainStatus::Running { ref mut health, .. }
| MainStatus::Starting { ref mut health } => {
MainStatus::Running { health, .. } | MainStatus::Starting { health } => {
health.insert(id, result);
}
_ => (),

View File

@@ -1,11 +1,11 @@
use std::net::Ipv4Addr;
use rpc_toolkit::{from_fn, from_fn_async, from_fn_blocking, Context, HandlerExt, ParentHandler};
use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn, from_fn_async, from_fn_blocking};
use crate::prelude::*;
use crate::service::cli::ContainerCliContext;
use crate::service::effects::context::EffectContext;
use crate::{echo, HOST_IP};
use crate::{HOST_IP, echo};
mod action;
pub mod callbacks;

View File

@@ -89,5 +89,6 @@ pub async fn get_service_port_forward(
.de()?
.get(&internal_port)
.or_not_found(lazy_format!("binding for port {internal_port}"))?
.net)
.net
.clone())
}

View File

@@ -6,11 +6,11 @@ use ipnet::IpNet;
use itertools::Itertools;
use openssl::pkey::{PKey, Private};
use crate::HOST_IP;
use crate::service::effects::callbacks::CallbackHandler;
use crate::service::effects::prelude::*;
use crate::service::rpc::CallbackId;
use crate::util::serde::Pem;
use crate::HOST_IP;
#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, TS, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
@@ -59,7 +59,8 @@ pub async fn get_ssl_certificate(
.de()?
.iter()
.map(InternedString::from_display)
.chain(m.as_domains().keys()?)
.chain(m.as_public_domains().keys()?)
.chain(m.as_private_domains().de()?)
.chain(
m.as_hostname_info()
.de()?
@@ -90,7 +91,7 @@ pub async fn get_ssl_certificate(
.as_public()
.as_server_info()
.as_network()
.as_network_interfaces()
.as_gateways()
.as_entries()?
.into_iter()
.flat_map(|(_, net)| net.as_ip_info().transpose_ref())
@@ -184,7 +185,8 @@ pub async fn get_ssl_key(
.de()?
.iter()
.map(InternedString::from_display)
.chain(m.as_domains().keys()?)
.chain(m.as_public_domains().keys()?)
.chain(m.as_private_domains().de()?)
.chain(
m.as_hostname_info()
.de()?

View File

@@ -9,8 +9,8 @@ pub(super) use crate::service::effects::context::EffectContext;
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct ProcedureId {
pub struct EventId {
#[serde(default)]
#[arg(default_value_t, long)]
pub procedure_id: Guid,
pub event_id: Guid,
}

View File

@@ -11,14 +11,14 @@ use crate::service::effects::prelude::*;
use crate::service::persistent_container::Subcontainer;
use crate::util::Invoke;
#[cfg(feature = "container-runtime")]
#[cfg(feature = "cli-container")]
mod sync;
#[cfg(not(feature = "container-runtime"))]
#[cfg(not(feature = "cli-container"))]
mod sync_dummy;
pub use sync::*;
#[cfg(not(feature = "container-runtime"))]
#[cfg(not(feature = "cli-container"))]
use sync_dummy as sync;
#[derive(Debug, Deserialize, Serialize, Parser, TS)]
@@ -61,7 +61,9 @@ pub async fn destroy_subcontainer_fs(
}
overlay.overlay.unmount(true).await?;
} else {
tracing::warn!("Could not find a subcontainer fs to destroy; assumming that it already is destroyed and will be skipping");
tracing::warn!(
"Could not find a subcontainer fs to destroy; assumming that it already is destroyed and will be skipping"
);
}
Ok(())
}

View File

@@ -1,5 +1,5 @@
use std::collections::BTreeMap;
use std::ffi::{c_int, OsStr, OsString};
use std::ffi::{OsStr, OsString, c_int};
use std::fs::File;
use std::io::{IsTerminal, Read};
use std::os::unix::process::{CommandExt, ExitStatusExt};
@@ -13,10 +13,10 @@ use signal_hook::consts::signal::*;
use termion::raw::IntoRawMode;
use tokio::sync::oneshot;
use crate::service::effects::prelude::*;
use crate::service::effects::ContainerCliContext;
use crate::util::io::TermSize;
use crate::CAP_1_KiB;
use crate::service::effects::ContainerCliContext;
use crate::service::effects::prelude::*;
use crate::util::io::TermSize;
const FWD_SIGNALS: &[c_int] = &[
SIGABRT, SIGALRM, SIGCONT, SIGHUP, SIGINT, SIGIO, SIGPIPE, SIGPROF, SIGQUIT, SIGTERM, SIGTRAP,
@@ -284,7 +284,7 @@ pub fn launch(
if tty {
use pty_process::blocking as pty_process;
let (pty, pts) = pty_process::open().with_kind(ErrorKind::Filesystem)?;
let mut cmd = pty_process::Command::new("/usr/bin/start-cli");
let mut cmd = pty_process::Command::new("/usr/bin/start-container");
cmd = cmd.arg("subcontainer").arg("launch-init");
if let Some(env) = env {
cmd = cmd.arg("--env").arg(env);
@@ -339,7 +339,7 @@ pub fn launch(
))
}
} else {
let mut cmd = StdCommand::new("/usr/bin/start-cli");
let mut cmd = StdCommand::new("/usr/bin/start-container");
cmd.arg("subcontainer").arg("launch-init");
if let Some(env) = env {
cmd.arg("--env").arg(env);
@@ -534,7 +534,7 @@ pub fn exec(
if tty {
use pty_process::blocking as pty_process;
let (pty, pts) = pty_process::open().with_kind(ErrorKind::Filesystem)?;
let mut cmd = pty_process::Command::new("/usr/bin/start-cli");
let mut cmd = pty_process::Command::new("/usr/bin/start-container");
cmd = cmd.arg("subcontainer").arg("exec-command");
if let Some(env) = env {
cmd = cmd.arg("--env").arg(env);
@@ -589,7 +589,7 @@ pub fn exec(
))
}
} else {
let mut cmd = StdCommand::new("/usr/bin/start-cli");
let mut cmd = StdCommand::new("/usr/bin/start-container");
cmd.arg("subcontainer").arg("exec-command");
if let Some(env) = env {
cmd.arg("--env").arg(env);

View File

@@ -1,5 +1,5 @@
use crate::service::effects::prelude::*;
use crate::service::effects::ContainerCliContext;
use crate::service::effects::prelude::*;
pub fn launch(_: ContainerCliContext) -> Result<(), Error> {
Err(Error::new(

View File

@@ -1,9 +1,9 @@
use std::path::Path;
use crate::DATA_DIR;
use crate::service::effects::prelude::*;
use crate::util::io::{delete_file, maybe_read_file_to_string, write_file_atomic};
use crate::volume::PKG_VOLUME_DIR;
use crate::DATA_DIR;
#[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)]
#[serde(rename_all = "camelCase")]

View File

@@ -15,12 +15,12 @@ use futures::future::BoxFuture;
use futures::stream::FusedStream;
use futures::{FutureExt, SinkExt, StreamExt, TryStreamExt};
use helpers::NonDetachingJoinHandle;
use imbl_value::{json, InternedString};
use imbl_value::{InternedString, json};
use itertools::Itertools;
use models::{ActionId, HostId, ImageId, PackageId};
use nix::sys::signal::Signal;
use persistent_container::{PersistentContainer, Subcontainer};
use rpc_toolkit::{from_fn_async, CallRemoteHandler, Empty, HandlerArgs, HandlerFor};
use rpc_toolkit::{CallRemoteHandler, Empty, HandlerArgs, HandlerFor, from_fn_async};
use serde::{Deserialize, Serialize};
use service_actor::ServiceActor;
use start_stop::StartStop;
@@ -47,11 +47,11 @@ use crate::service::action::update_tasks;
use crate::service::rpc::{ExitParams, InitKind};
use crate::service::service_map::InstallProgressHandles;
use crate::service::uninstall::cleanup;
use crate::util::Never;
use crate::util::actor::concurrent::ConcurrentActor;
use crate::util::io::{create_file, delete_file, AsyncReadStream, TermSize};
use crate::util::io::{AsyncReadStream, TermSize, create_file, delete_file};
use crate::util::net::WebSocketExt;
use crate::util::serde::Pem;
use crate::util::Never;
use crate::volume::data_dir;
use crate::{CAP_1_KiB, DATA_DIR};
@@ -852,7 +852,9 @@ pub async fn attach(
.map(format_subcontainer_pair)
.join("\n");
return Err(Error::new(
eyre!("no matching subcontainers are running for {id}; some possible choices are:\n{subcontainers}"),
eyre!(
"no matching subcontainers are running for {id}; some possible choices are:\n{subcontainers}"
),
ErrorKind::NotFound,
));
};
@@ -910,7 +912,7 @@ pub async fn attach(
cmd.arg(&*container_id)
.arg("--")
.arg("start-cli")
.arg("start-container")
.arg("subcontainer")
.arg("exec")
.arg("--env")

View File

@@ -4,16 +4,16 @@ use std::path::Path;
use std::sync::{Arc, Weak};
use std::time::Duration;
use futures::future::ready;
use futures::Future;
use futures::future::ready;
use helpers::NonDetachingJoinHandle;
use imbl::{vector, Vector};
use imbl::{Vector, vector};
use imbl_value::InternedString;
use models::{ImageId, ProcedureName, VolumeId};
use rpc_toolkit::{Empty, Server, ShutdownHandle};
use serde::de::DeserializeOwned;
use tokio::process::Command;
use tokio::sync::{oneshot, watch, Mutex, OnceCell};
use tokio::sync::{Mutex, OnceCell, oneshot, watch};
use tracing::instrument;
use crate::context::RpcContext;
@@ -23,12 +23,12 @@ use crate::disk::mount::filesystem::loop_dev::LoopDev;
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
use crate::disk::mount::filesystem::{MountType, ReadOnly};
use crate::disk::mount::guard::{GenericMountGuard, MountGuard};
use crate::lxc::{LxcConfig, LxcContainer, HOST_RPC_SERVER_SOCKET};
use crate::lxc::{HOST_RPC_SERVER_SOCKET, LxcConfig, LxcContainer};
use crate::net::net_controller::NetService;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::service::effects::context::EffectContext;
use crate::service::effects::handler;
use crate::service::rpc::{
@@ -36,10 +36,10 @@ use crate::service::rpc::{
};
use crate::service::start_stop::StartStop;
use crate::service::transition::{TransitionKind, TransitionState};
use crate::service::{rpc, RunningStatus, Service};
use crate::service::{RunningStatus, Service, rpc};
use crate::util::Invoke;
use crate::util::io::create_file;
use crate::util::rpc_client::UnixRpcClient;
use crate::util::Invoke;
use crate::volume::data_dir;
use crate::{ARCH, DATA_DIR, PACKAGE_DATA};
@@ -105,7 +105,6 @@ pub struct PersistentContainer {
pub(super) lxc_container: OnceCell<LxcContainer>,
pub(super) rpc_client: UnixRpcClient,
pub(super) rpc_server: watch::Sender<Option<(NonDetachingJoinHandle<()>, ShutdownHandle)>>,
// procedures: Mutex<Vec<(ProcedureName, ProcedureId)>>,
js_mount: MountGuard,
volumes: BTreeMap<VolumeId, MountGuard>,
assets: Vec<MountGuard>,

View File

@@ -8,8 +8,8 @@ use exver::{ExtendedVersion, VersionRange};
use imbl::Vector;
use imbl_value::{InternedString, Value};
use models::{FromStrParser, ProcedureName};
use rpc_toolkit::yajrc::RpcMethod;
use rpc_toolkit::Empty;
use rpc_toolkit::yajrc::RpcMethod;
use ts_rs::TS;
use crate::prelude::*;

View File

@@ -3,15 +3,15 @@ use std::time::Duration;
use imbl::vector;
use super::start_stop::StartStop;
use super::ServiceActorSeed;
use super::start_stop::StartStop;
use crate::prelude::*;
use crate::service::SYNC_RETRY_COOLDOWN_SECONDS;
use crate::service::persistent_container::ServiceStateKinds;
use crate::service::transition::TransitionKind;
use crate::service::SYNC_RETRY_COOLDOWN_SECONDS;
use crate::status::MainStatus;
use crate::util::actor::background::BackgroundJobQueue;
use crate::util::actor::Actor;
use crate::util::actor::background::BackgroundJobQueue;
#[derive(Clone)]
pub(super) struct ServiceActor(pub(super) Arc<ServiceActorSeed>);

View File

@@ -10,30 +10,30 @@ use futures::{Future, FutureExt, StreamExt, TryFutureExt};
use helpers::NonDetachingJoinHandle;
use imbl::OrdMap;
use models::ErrorData;
use tokio::sync::{oneshot, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock};
use tokio::sync::{OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock, oneshot};
use tracing::instrument;
use url::Url;
use crate::DATA_DIR;
use crate::context::RpcContext;
use crate::db::model::package::{
InstallingInfo, InstallingState, PackageDataEntry, PackageState, UpdatingState,
};
use crate::disk::mount::guard::GenericMountGuard;
use crate::install::PKG_ARCHIVE_DIR;
use crate::notifications::{notify, NotificationLevel};
use crate::notifications::{NotificationLevel, notify};
use crate::prelude::*;
use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle, ProgressTrackerWriter};
use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment;
use crate::s9pk::S9pk;
use crate::s9pk::manifest::PackageId;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
use crate::service::rpc::ExitParams;
use crate::service::start_stop::StartStop;
use crate::service::{LoadDisposition, Service, ServiceRef};
use crate::sign::commitment::merkle_archive::MerkleArchiveCommitment;
use crate::status::MainStatus;
use crate::util::serde::{Base32, Pem};
use crate::util::sync::SyncMutex;
use crate::DATA_DIR;
pub type DownloadInstallFuture = BoxFuture<'static, Result<InstallFuture, Error>>;
pub type InstallFuture = BoxFuture<'static, Result<(), Error>>;
@@ -382,7 +382,7 @@ impl ServiceMap {
id: PackageId,
soft: bool,
force: bool,
) -> Result<impl Future<Output = Result<(), Error>> + Send, Error> {
) -> Result<impl Future<Output = Result<(), Error>> + Send + 'static, Error> {
let mut guard = self.get_mut(&id).await;
ctx.db
.mutate(|db| {

View File

@@ -1,17 +1,17 @@
use std::path::PathBuf;
use std::sync::Arc;
use futures::future::BoxFuture;
use futures::FutureExt;
use futures::future::BoxFuture;
use models::ProcedureName;
use super::TempDesiredRestore;
use crate::disk::mount::filesystem::ReadWrite;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::ServiceActor;
use crate::service::action::GetActionInput;
use crate::service::transition::{TransitionKind, TransitionState};
use crate::service::ServiceActor;
use crate::util::actor::background::BackgroundJobQueue;
use crate::util::actor::{ConflictBuilder, Handler};
use crate::util::future::RemoteCancellable;

View File

@@ -1,5 +1,5 @@
use futures::future::BoxFuture;
use futures::FutureExt;
use futures::future::BoxFuture;
use super::TempDesiredRestore;
use crate::prelude::*;