Feature/cli clearnet (#2789)

* add support for ACME cert acquisition

* add support for modifying hosts for a package

* misc fixes

* more fixes

* use different port for lan clearnet than wan clearnet

* fix chroot-and-upgrade always growing

* bail on failure

* wip

* fix alpn auth

* bump async-acme

* fix cli

* add barebones documentation

* add domain to hostname info

CLEARNET.md

@@ -0,0 +1,40 @@
+# Setting up clearnet for a service interface
+
+NOTE: this guide is for HTTPS only! Other configurations may require a more bespoke setup depending on the service. Please consult the service documentation or the Start9 Community for help with non-HTTPS applications
+
+## Initialize ACME certificate generation
+
+The following command will register your device with an ACME certificate provider, such as letsencrypt
+
+This only needs to be done once.
+
+```
+start-cli net acme init --provider=letsencrypt --contact="mailto:me@drbonez.dev"
+```
+
+- `provider` can be `letsencrypt`, `letsencrypt-staging` (useful if you're doing a lot of testing and want to avoid being rate limited), or the url of any provider that supports the [RFC8555](https://datatracker.ietf.org/doc/html/rfc8555) ACME api
+- `contact` can be any valid contact url, typically `mailto:` urls. it can be specified multiple times to set multiple contacts
+
+## Whitelist a domain for ACME certificate acquisition
+
+The following command will tell the OS to use ACME certificates instead of system signed ones for the provided url. In this example, `testing.drbonez.dev`
+
+This must be done for every domain you wish to host on clearnet.
+
+```
+start-cli net acme domain add "testing.drbonez.dev"
+```
+
+## Forward clearnet port
+
+Go into your router settings, and map port 443 on your router to port 5443 on your start-os device. This one port should cover most use cases
+
+## Add domain to service host
+
+The following command will tell the OS to route https requests from the WAN to the provided hostname to the specified service. In this example, we are adding `testing.drbonez.dev` to the host `ui-multi` on the package `hello-world`. To see a list of available host IDs for a given package, run `start-cli package host <PACKAGE> list`
+
+This must be done for every domain you wish to host on clearnet.
+
+```
+start-cli package host hello-world address ui-multi add testing.drbonez.dev
+```

build/lib/scripts/chroot-and-upgrade

@@ -77,6 +77,7 @@ umount /media/startos/next/dev
 umount /media/startos/next/sys
 umount /media/startos/next/proc
 umount /media/startos/next/boot
+umount /media/startos/next/media/startos/root
 
 if [ "$CHROOT_RES" -eq 0 ]; then
 
@@ -86,7 +87,12 @@ if [ "$CHROOT_RES" -eq 0 ]; then
 
     echo 'Upgrading...'
 
-    time mksquashfs /media/startos/next /media/startos/images/next.squashfs -b 4096 -comp gzip
+    if ! time mksquashfs /media/startos/next /media/startos/images/next.squashfs -b 4096 -comp gzip; then
+      umount -R /media/startos/next
+      umount -R /media/startos/upper
+      rm -rf /media/startos/upper /media/startos/next
+      exit 1
+    fi
     hash=$(b3sum /media/startos/images/next.squashfs | head -c 32)
     mv /media/startos/images/next.squashfs /media/startos/images/${hash}.rootfs
     ln -rsf /media/startos/images/${hash}.rootfs /media/startos/config/current.rootfs

build/lib/scripts/prune-images

@@ -33,10 +33,11 @@ if [ -h /media/startos/config/current.rootfs ] && [ -e /media/startos/config/cur
     echo 'Pruning...'
     current="$(readlink -f /media/startos/config/current.rootfs)"
     while [[ "$(df -B1 --output=avail --sync /media/startos/images | tail -n1)" -lt "$needed" ]]; do
-        to_prune="$(ls -t1 /media/startos/images/*.rootfs /media/startos/images/*.squashfs | grep -v "$current" | tail -n1)"
+        to_prune="$(ls -t1 /media/startos/images/*.rootfs /media/startos/images/*.squashfs 2> /dev/null | grep -v "$current" | tail -n1)"
         if [ -e "$to_prune" ]; then
             echo "  Pruning $to_prune"
             rm -rf "$to_prune"
+            sync
         else
             >&2 echo "Not enough space and nothing to prune!"
             exit 1

core/models/src/errors.rs

@@ -322,6 +322,11 @@ impl From<reqwest::Error> for Error {
         Error::new(e, kind)
     }
 }
+impl From<torut::onion::OnionAddressParseError> for Error {
+    fn from(e: torut::onion::OnionAddressParseError) -> Self {
+        Error::new(e, ErrorKind::Tor)
+    }
+}
 impl From<patch_db::value::Error> for Error {
     fn from(value: patch_db::value::Error) -> Self {
         match value.kind {

core/startos/Cargo.toml

@@ -50,6 +50,10 @@ test = []
 
 [dependencies]
 aes = { version = "0.7.5", features = ["ctr"] }
+async-acme = { version = "0.5.0", git = "https://github.com/dr-bonez/async-acme.git", features = [
+  "use_rustls",
+  "use_tokio",
+] }
 async-compression = { version = "0.4.4", features = [
   "gzip",
   "brotli",

core/startos/src/db/model/public.rs

@@ -55,6 +55,7 @@ impl Public {
                     .parse()
                     .unwrap(),
                 ip_info: BTreeMap::new(),
+                acme: None,
                 status_info: ServerStatus {
                     backup_progress: None,
                     updated: false,
@@ -130,6 +131,7 @@ pub struct ServerInfo {
     #[ts(type = "string")]
     pub tor_address: Url,
     pub ip_info: BTreeMap<String, IpInfo>,
+    pub acme: Option<AcmeSettings>,
     #[serde(default)]
     pub status_info: ServerStatus,
     pub wifi: WifiInfo,
@@ -174,6 +176,20 @@ impl IpInfo {
     }
 }
 
+#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
+#[serde(rename_all = "camelCase")]
+#[model = "Model<Self>"]
+#[ts(export)]
+pub struct AcmeSettings {
+    #[ts(type = "string")]
+    pub provider: Url,
+    /// email addresses for letsencrypt
+    pub contact: Vec<String>,
+    #[ts(type = "string[]")]
+    /// domains to get letsencrypt certs for
+    pub domains: BTreeSet<InternedString>,
+}
+
 #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)]
 #[model = "Model<Self>"]
 #[ts(export)]

core/startos/src/lib.rs

@@ -355,7 +355,7 @@ pub fn package<C: Context>() -> ParentHandler<C> {
             from_fn_async(control::start)
                 .with_metadata("sync_db", Value::Bool(true))
                 .no_display()
-                .with_about("Start a package container")
+                .with_about("Start a service")
                 .with_call_remote::<CliContext>(),
         )
         .subcommand(
@@ -363,7 +363,7 @@ pub fn package<C: Context>() -> ParentHandler<C> {
             from_fn_async(control::stop)
                 .with_metadata("sync_db", Value::Bool(true))
                 .no_display()
-                .with_about("Stop a package container")
+                .with_about("Stop a service")
                 .with_call_remote::<CliContext>(),
         )
         .subcommand(
@@ -371,7 +371,7 @@ pub fn package<C: Context>() -> ParentHandler<C> {
             from_fn_async(control::restart)
                 .with_metadata("sync_db", Value::Bool(true))
                 .no_display()
-                .with_about("Restart a package container")
+                .with_about("Restart a service")
                 .with_call_remote::<CliContext>(),
         )
         .subcommand(
@@ -409,9 +409,14 @@ pub fn package<C: Context>() -> ParentHandler<C> {
             "attach",
             from_fn_async(service::attach)
                 .with_metadata("get_session", Value::Bool(true))
+                .with_about("Execute commands within a service container")
                 .no_cli(),
         )
         .subcommand("attach", from_fn_async(service::cli_attach).no_display())
+        .subcommand(
+            "host",
+            net::host::host::<C>().with_about("Manage network hosts for a package"),
+        )
 }
 
 pub fn diagnostic_api() -> ParentHandler<DiagnosticContext> {

core/startos/src/net/acme.rs

@@ -0,0 +1,324 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::str::FromStr;
+
+use clap::builder::ValueParserFactory;
+use clap::Parser;
+use imbl_value::InternedString;
+use itertools::Itertools;
+use models::{ErrorData, FromStrParser};
+use openssl::pkey::{PKey, Private};
+use openssl::x509::X509;
+use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+use crate::context::{CliContext, RpcContext};
+use crate::db::model::public::AcmeSettings;
+use crate::db::model::Database;
+use crate::prelude::*;
+use crate::util::serde::{Pem, Pkcs8Doc};
+
+#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
+#[model = "Model<Self>"]
+pub struct AcmeCertStore {
+    pub accounts: BTreeMap<JsonKey<Vec<String>>, Pem<Pkcs8Doc>>,
+    pub certs: BTreeMap<Url, BTreeMap<JsonKey<BTreeSet<InternedString>>, AcmeCert>>,
+}
+impl AcmeCertStore {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct AcmeCert {
+    pub key: Pem<PKey<Private>>,
+    pub fullchain: Vec<Pem<X509>>,
+}
+
+pub struct AcmeCertCache<'a>(pub &'a TypedPatchDb<Database>);
+#[async_trait::async_trait]
+impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
+    type Error = ErrorData;
+
+    async fn read_account(&self, contacts: &[&str]) -> Result<Option<Vec<u8>>, Self::Error> {
+        let contacts = JsonKey::new(contacts.into_iter().map(|s| (*s).to_owned()).collect_vec());
+        let Some(account) = self
+            .0
+            .peek()
+            .await
+            .into_private()
+            .into_key_store()
+            .into_acme()
+            .into_accounts()
+            .into_idx(&contacts)
+        else {
+            return Ok(None);
+        };
+        Ok(Some(account.de()?.0.document.into_vec()))
+    }
+
+    async fn write_account(&self, contacts: &[&str], contents: &[u8]) -> Result<(), Self::Error> {
+        let contacts = JsonKey::new(contacts.into_iter().map(|s| (*s).to_owned()).collect_vec());
+        let key = Pkcs8Doc {
+            tag: "EC PRIVATE KEY".into(),
+            document: pkcs8::Document::try_from(contents).with_kind(ErrorKind::Pem)?,
+        };
+        self.0
+            .mutate(|db| {
+                db.as_private_mut()
+                    .as_key_store_mut()
+                    .as_acme_mut()
+                    .as_accounts_mut()
+                    .insert(&contacts, &Pem::new(key))
+            })
+            .await?;
+        Ok(())
+    }
+
+    async fn read_certificate(
+        &self,
+        domains: &[String],
+        directory_url: &str,
+    ) -> Result<Option<(String, String)>, Self::Error> {
+        let domains = JsonKey::new(domains.into_iter().map(InternedString::intern).collect());
+        let directory_url = directory_url
+            .parse::<Url>()
+            .with_kind(ErrorKind::ParseUrl)?;
+        let Some(cert) = self
+            .0
+            .peek()
+            .await
+            .into_private()
+            .into_key_store()
+            .into_acme()
+            .into_certs()
+            .into_idx(&directory_url)
+            .and_then(|a| a.into_idx(&domains))
+        else {
+            return Ok(None);
+        };
+        let cert = cert.de()?;
+        Ok(Some((
+            String::from_utf8(
+                cert.key
+                    .0
+                    .private_key_to_pem_pkcs8()
+                    .with_kind(ErrorKind::OpenSsl)?,
+            )
+            .with_kind(ErrorKind::Utf8)?,
+            cert.fullchain
+                .into_iter()
+                .map(|cert| {
+                    String::from_utf8(cert.0.to_pem().with_kind(ErrorKind::OpenSsl)?)
+                        .with_kind(ErrorKind::Utf8)
+                })
+                .collect::<Result<Vec<_>, _>>()?
+                .join("\n"),
+        )))
+    }
+
+    async fn write_certificate(
+        &self,
+        domains: &[String],
+        directory_url: &str,
+        key_pem: &str,
+        certificate_pem: &str,
+    ) -> Result<(), Self::Error> {
+        tracing::info!("Saving new certificate for {domains:?}");
+        let domains = JsonKey::new(domains.into_iter().map(InternedString::intern).collect());
+        let directory_url = directory_url
+            .parse::<Url>()
+            .with_kind(ErrorKind::ParseUrl)?;
+        let cert = AcmeCert {
+            key: Pem(PKey::<Private>::private_key_from_pem(key_pem.as_bytes())
+                .with_kind(ErrorKind::OpenSsl)?),
+            fullchain: X509::stack_from_pem(certificate_pem.as_bytes())
+                .with_kind(ErrorKind::OpenSsl)?
+                .into_iter()
+                .map(Pem)
+                .collect(),
+        };
+        self.0
+            .mutate(|db| {
+                db.as_private_mut()
+                    .as_key_store_mut()
+                    .as_acme_mut()
+                    .as_certs_mut()
+                    .upsert(&directory_url, || Ok(BTreeMap::new()))?
+                    .insert(&domains, &cert)
+            })
+            .await?;
+
+        Ok(())
+    }
+}
+
+pub fn acme<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "init",
+            from_fn_async(init)
+                .no_display()
+                .with_about("Setup ACME certificate acquisition")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "domain",
+            domain::<C>()
+                .with_about("Add, remove, or view domains for which to acquire ACME certificates"),
+        )
+}
+
+#[derive(Clone, Deserialize, Serialize)]
+pub struct AcmeProvider(pub Url);
+impl FromStr for AcmeProvider {
+    type Err = <Url as FromStr>::Err;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "letsencrypt" => async_acme::acme::LETS_ENCRYPT_PRODUCTION_DIRECTORY.parse(),
+            "letsencrypt-staging" => async_acme::acme::LETS_ENCRYPT_STAGING_DIRECTORY.parse(),
+            s => s.parse(),
+        }
+        .map(Self)
+    }
+}
+impl ValueParserFactory for AcmeProvider {
+    type Parser = FromStrParser<Self>;
+    fn value_parser() -> Self::Parser {
+        Self::Parser::new()
+    }
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct InitAcmeParams {
+    #[arg(long)]
+    pub provider: AcmeProvider,
+    #[arg(long)]
+    pub contact: Vec<String>,
+}
+
+pub async fn init(
+    ctx: RpcContext,
+    InitAcmeParams {
+        provider: AcmeProvider(provider),
+        contact,
+    }: InitAcmeParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            db.as_public_mut()
+                .as_server_info_mut()
+                .as_acme_mut()
+                .map_mutate(|acme| {
+                    Ok(Some(AcmeSettings {
+                        provider,
+                        contact,
+                        domains: acme.map(|acme| acme.domains).unwrap_or_default(),
+                    }))
+                })
+        })
+        .await?;
+    Ok(())
+}
+
+pub fn domain<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "add",
+            from_fn_async(add_domain)
+                .no_display()
+                .with_about("Add a domain for which to acquire ACME certificates")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "remove",
+            from_fn_async(remove_domain)
+                .no_display()
+                .with_about("Remove a domain for which to acquire ACME certificates")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "list",
+            from_fn_async(list_domains)
+                .with_custom_display_fn(|_, res| {
+                    for domain in res {
+                        println!("{domain}")
+                    }
+                    Ok(())
+                })
+                .with_about("List domains for which to acquire ACME certificates")
+                .with_call_remote::<CliContext>(),
+        )
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct DomainParams {
+    pub domain: InternedString,
+}
+
+pub async fn add_domain(
+    ctx: RpcContext,
+    DomainParams { domain }: DomainParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            db.as_public_mut()
+                .as_server_info_mut()
+                .as_acme_mut()
+                .transpose_mut()
+                .ok_or_else(|| {
+                    Error::new(
+                        eyre!("Please call `start-cli net acme init` before adding a domain"),
+                        ErrorKind::InvalidRequest,
+                    )
+                })?
+                .as_domains_mut()
+                .mutate(|domains| {
+                    domains.insert(domain);
+                    Ok(())
+                })
+        })
+        .await?;
+    Ok(())
+}
+
+pub async fn remove_domain(
+    ctx: RpcContext,
+    DomainParams { domain }: DomainParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            if let Some(acme) = db
+                .as_public_mut()
+                .as_server_info_mut()
+                .as_acme_mut()
+                .transpose_mut()
+            {
+                acme.as_domains_mut().mutate(|domains| {
+                    domains.remove(&domain);
+                    Ok(())
+                })
+            } else {
+                Ok(())
+            }
+        })
+        .await?;
+    Ok(())
+}
+
+pub async fn list_domains(ctx: RpcContext) -> Result<BTreeSet<InternedString>, Error> {
+    if let Some(acme) = ctx
+        .db
+        .peek()
+        .await
+        .into_public()
+        .into_server_info()
+        .into_acme()
+        .transpose()
+    {
+        acme.into_domains().de()
+    } else {
+        Ok(BTreeSet::new())
+    }
+}

core/startos/src/net/host/address.rs

@@ -1,7 +1,9 @@
 use std::fmt;
 use std::str::FromStr;
 
+use clap::builder::ValueParserFactory;
 use imbl_value::InternedString;
+use models::FromStrParser;
 use serde::{Deserialize, Serialize};
 use torut::onion::OnionAddressV3;
 use ts_rs::TS;
@@ -46,3 +48,10 @@ impl fmt::Display for HostAddress {
         }
     }
 }
+
+impl ValueParserFactory for HostAddress {
+    type Parser = FromStrParser<Self>;
+    fn value_parser() -> Self::Parser {
+        Self::Parser::new()
+    }
+}

core/startos/src/net/host/mod.rs

@@ -1,10 +1,13 @@
 use std::collections::{BTreeMap, BTreeSet};
 
+use clap::Parser;
 use imbl_value::InternedString;
 use models::{HostId, PackageId};
+use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
 
+use crate::context::{CliContext, RpcContext};
 use crate::db::model::DatabaseModel;
 use crate::net::forward::AvailablePorts;
 use crate::net::host::address::HostAddress;
@@ -134,3 +137,163 @@ impl Model<Host> {
         })
     }
 }
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct HostParams {
+    package: PackageId,
+}
+
+pub fn host<C: Context>() -> ParentHandler<C, HostParams> {
+    ParentHandler::<C, HostParams>::new()
+        .subcommand(
+            "list",
+            from_fn_async(list_hosts)
+                .with_inherited(|HostParams { package }, _| package)
+                .with_custom_display_fn(|_, ids| {
+                    for id in ids {
+                        println!("{id}")
+                    }
+                    Ok(())
+                })
+                .with_about("List host IDs available for this service"),
+        )
+        .subcommand(
+            "address",
+            address::<C>().with_inherited(|HostParams { package }, _| package),
+        )
+}
+
+pub async fn list_hosts(
+    ctx: RpcContext,
+    _: Empty,
+    package: PackageId,
+) -> Result<Vec<HostId>, Error> {
+    ctx.db
+        .peek()
+        .await
+        .into_public()
+        .into_package_data()
+        .into_idx(&package)
+        .or_not_found(&package)?
+        .into_hosts()
+        .keys()
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct AddressApiParams {
+    host: HostId,
+}
+
+pub fn address<C: Context>() -> ParentHandler<C, AddressApiParams, PackageId> {
+    ParentHandler::<C, AddressApiParams, PackageId>::new()
+        .subcommand(
+            "add",
+            from_fn_async(add_address)
+                .with_inherited(|AddressApiParams { host }, package| (package, host))
+                .no_display()
+                .with_about("Add an address to this host")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "remove",
+            from_fn_async(remove_address)
+                .with_inherited(|AddressApiParams { host }, package| (package, host))
+                .no_display()
+                .with_about("Remove an address from this host")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "list",
+            from_fn_async(list_addresses)
+                .with_inherited(|AddressApiParams { host }, package| (package, host))
+                .with_custom_display_fn(|_, res| {
+                    for address in res {
+                        println!("{address}")
+                    }
+                    Ok(())
+                })
+                .with_about("List addresses for this host")
+                .with_call_remote::<CliContext>(),
+        )
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct AddressParams {
+    pub address: HostAddress,
+}
+
+pub async fn add_address(
+    ctx: RpcContext,
+    AddressParams { address }: AddressParams,
+    (package, host): (PackageId, HostId),
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            if let HostAddress::Onion { address } = address {
+                db.as_private()
+                    .as_key_store()
+                    .as_onion()
+                    .get_key(&address)?;
+            }
+
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(&package)
+                .or_not_found(&package)?
+                .as_hosts_mut()
+                .as_idx_mut(&host)
+                .or_not_found(&host)?
+                .as_addresses_mut()
+                .mutate(|a| Ok(a.insert(address)))
+        })
+        .await?;
+    let service = ctx.services.get(&package).await;
+    let service_ref = service.as_ref().or_not_found(&package)?;
+    service_ref.update_host(host).await?;
+
+    Ok(())
+}
+
+pub async fn remove_address(
+    ctx: RpcContext,
+    AddressParams { address }: AddressParams,
+    (package, host): (PackageId, HostId),
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(&package)
+                .or_not_found(&package)?
+                .as_hosts_mut()
+                .as_idx_mut(&host)
+                .or_not_found(&host)?
+                .as_addresses_mut()
+                .mutate(|a| Ok(a.remove(&address)))
+        })
+        .await?;
+    let service = ctx.services.get(&package).await;
+    let service_ref = service.as_ref().or_not_found(&package)?;
+    service_ref.update_host(host).await?;
+
+    Ok(())
+}
+
+pub async fn list_addresses(
+    ctx: RpcContext,
+    _: Empty,
+    (package, host): (PackageId, HostId),
+) -> Result<BTreeSet<HostAddress>, Error> {
+    ctx.db
+        .peek()
+        .await
+        .into_public()
+        .into_package_data()
+        .into_idx(&package)
+        .or_not_found(&package)?
+        .into_hosts()
+        .into_idx(&host)
+        .or_not_found(&host)?
+        .into_addresses()
+        .de()
+}

core/startos/src/net/keys.rs

@@ -1,6 +1,7 @@
 use serde::{Deserialize, Serialize};
 
 use crate::account::AccountInfo;
+use crate::net::acme::AcmeCertStore;
 use crate::net::ssl::CertStore;
 use crate::net::tor::OnionStore;
 use crate::prelude::*;
@@ -10,13 +11,15 @@ use crate::prelude::*;
 pub struct KeyStore {
     pub onion: OnionStore,
     pub local_certs: CertStore,
-    // pub letsencrypt_certs: BTreeMap<BTreeSet<InternedString>, CertData>
+    #[serde(default)]
+    pub acme: AcmeCertStore,
 }
 impl KeyStore {
     pub fn new(account: &AccountInfo) -> Result<Self, Error> {
         let mut res = Self {
             onion: OnionStore::new(),
             local_certs: CertStore::new(account)?,
+            acme: AcmeCertStore::new(),
         };
         res.onion.insert(account.tor_key.clone());
         Ok(res)

core/startos/src/net/mod.rs

@@ -1,5 +1,6 @@
 use rpc_toolkit::{Context, HandlerExt, ParentHandler};
 
+pub mod acme;
 pub mod dhcp;
 pub mod dns;
 pub mod forward;
@@ -28,4 +29,8 @@ pub fn net<C: Context>() -> ParentHandler<C> {
             "dhcp",
             dhcp::dhcp::<C>().with_about("Command to update IP assigned from dhcp"),
         )
+        .subcommand(
+            "acme",
+            acme::acme::<C>().with_about("Setup automatic clearnet certificate acquisition"),
+        )
 }

core/startos/src/net/net_controller.rs

@@ -261,7 +261,7 @@ impl NetService {
         errors.into_result()
     }
 
-    async fn update(&mut self, id: HostId, host: Host) -> Result<(), Error> {
+    pub async fn update(&mut self, id: HostId, host: Host) -> Result<(), Error> {
         let ctrl = self.net_controller()?;
         let mut hostname_info = BTreeMap::new();
         let binds = self.binds.entry(id.clone()).or_default();
@@ -330,16 +330,29 @@ impl NetService {
                                 }
                                 HostAddress::Domain { address } => {
                                     if hostnames.insert(address.clone()) {
+                                        let address = Some(address.clone());
                                         rcs.push(
                                             ctrl.vhost
                                                 .add(
-                                                    Some(address.clone()),
+                                                    address.clone(),
                                                     external,
                                                     target,
                                                     connect_ssl.clone(),
                                                 )
                                                 .await?,
                                         );
+                                        if ssl.preferred_external_port == 443 {
+                                            rcs.push(
+                                                ctrl.vhost
+                                                    .add(
+                                                        address.clone(),
+                                                        5443,
+                                                        target,
+                                                        connect_ssl.clone(),
+                                                    )
+                                                    .await?,
+                                            );
+                                        }
                                     }
                                 }
                             }
@@ -363,11 +376,32 @@ impl NetService {
                         network_interface_id: interface.clone(),
                         public: false,
                         hostname: IpHostname::Local {
-                            value: format!("{hostname}.local"),
+                            value: InternedString::from_display(&{
+                                let hostname = &hostname;
+                                lazy_format!("{hostname}.local")
+                            }),
                             port: new_lan_bind.0.assigned_port,
                             ssl_port: new_lan_bind.0.assigned_ssl_port,
                         },
                     });
+                    for address in host.addresses() {
+                        if let HostAddress::Domain { address } = address {
+                            if let Some(ssl) = &new_lan_bind.1 {
+                                if ssl.preferred_external_port == 443 {
+                                    bind_hostname_info.push(HostnameInfo::Ip {
+                                        network_interface_id: interface.clone(),
+                                        public: false,
+                                        hostname: IpHostname::Domain {
+                                            domain: address.clone(),
+                                            subdomain: None,
+                                            port: None,
+                                            ssl_port: Some(443),
+                                        },
+                                    });
+                                }
+                            }
+                        }
+                    }
                     if let Some(ipv4) = ip_info.ipv4 {
                         bind_hostname_info.push(HostnameInfo::Ip {
                             network_interface_id: interface.clone(),
@@ -515,6 +549,7 @@ impl NetService {
                 ctrl.tor.gc(Some(addr.clone()), None).await?;
             }
         }
+
         self.net_controller()?
             .db
             .mutate(|db| {

core/startos/src/net/service_interface.rs

@@ -47,13 +47,16 @@ pub enum IpHostname {
         ssl_port: Option<u16>,
     },
     Local {
-        value: String,
+        #[ts(type = "string")]
+        value: InternedString,
         port: Option<u16>,
         ssl_port: Option<u16>,
     },
     Domain {
-        domain: String,
+        #[ts(type = "string")]
-        subdomain: Option<String>,
+        domain: InternedString,
+        #[ts(type = "string | null")]
+        subdomain: Option<InternedString>,
         port: Option<u16>,
         ssl_port: Option<u16>,
     },

core/startos/src/net/tor.rs

@@ -26,7 +26,7 @@ use ts_rs::TS;
 use crate::context::{CliContext, RpcContext};
 use crate::logs::{journalctl, LogSource, LogsParams};
 use crate::prelude::*;
-use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
+use crate::util::serde::{display_serializable, Base64, HandlerExtSerde, WithIoFormat};
 use crate::util::Invoke;
 
 pub const SYSTEMD_UNIT: &str = "tor@default";
@@ -59,7 +59,9 @@ impl Model<OnionStore> {
         self.insert(&key.public().get_onion_address(), &key)
     }
     pub fn get_key(&self, address: &OnionAddressV3) -> Result<TorSecretKeyV3, Error> {
-        self.as_idx(address).or_not_found(address)?.de()
+        self.as_idx(address)
+            .or_not_found(lazy_format!("private key for {address}"))?
+            .de()
     }
 }
 
@@ -108,7 +110,85 @@ pub fn tor<C: Context>() -> ParentHandler<C> {
                 .with_about("Reset Tor daemon")
                 .with_call_remote::<CliContext>(),
         )
+        .subcommand(
+            "key",
+            key::<C>().with_about("Manage the onion service key store"),
+        )
 }
+
+pub fn key<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "generate",
+            from_fn_async(generate_key)
+                .with_about("Generate an onion service key and add it to the key store")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "add",
+            from_fn_async(add_key)
+                .with_about("Add an onion service key to the key store")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "list",
+            from_fn_async(list_keys)
+                .with_custom_display_fn(|_, res| {
+                    for addr in res {
+                        println!("{addr}");
+                    }
+                    Ok(())
+                })
+                .with_about("List onion services with keys in the key store")
+                .with_call_remote::<CliContext>(),
+        )
+}
+
+pub async fn generate_key(ctx: RpcContext) -> Result<OnionAddressV3, Error> {
+    ctx.db
+        .mutate(|db| {
+            Ok(db
+                .as_private_mut()
+                .as_key_store_mut()
+                .as_onion_mut()
+                .new_key()?
+                .public()
+                .get_onion_address())
+        })
+        .await
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct AddKeyParams {
+    pub key: Base64<[u8; 64]>,
+}
+
+pub async fn add_key(
+    ctx: RpcContext,
+    AddKeyParams { key }: AddKeyParams,
+) -> Result<OnionAddressV3, Error> {
+    let key = TorSecretKeyV3::from(key.0);
+    ctx.db
+        .mutate(|db| {
+            db.as_private_mut()
+                .as_key_store_mut()
+                .as_onion_mut()
+                .insert_key(&key)
+        })
+        .await?;
+    Ok(key.public().get_onion_address())
+}
+
+pub async fn list_keys(ctx: RpcContext) -> Result<Vec<OnionAddressV3>, Error> {
+    ctx.db
+        .peek()
+        .await
+        .into_private()
+        .into_key_store()
+        .into_onion()
+        .keys()
+}
+
 #[derive(Deserialize, Serialize, Parser, TS)]
 #[serde(rename_all = "camelCase")]
 #[command(rename_all = "kebab-case")]

core/startos/src/net/vhost.rs

@@ -4,6 +4,7 @@ use std::str::FromStr;
 use std::sync::{Arc, Weak};
 use std::time::Duration;
 
+use async_acme::acme::ACME_TLS_ALPN_NAME;
 use axum::body::Body;
 use axum::extract::Request;
 use axum::response::Response;
@@ -15,31 +16,47 @@ use models::ResultExt;
 use serde::{Deserialize, Serialize};
 use tokio::io::AsyncWriteExt;
 use tokio::net::{TcpListener, TcpStream};
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::{watch, Mutex, RwLock};
+use tokio_rustls::rustls::crypto::CryptoProvider;
 use tokio_rustls::rustls::pki_types::{
     CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName,
 };
-use tokio_rustls::rustls::server::Acceptor;
+use tokio_rustls::rustls::server::{Acceptor, ResolvesServerCert};
+use tokio_rustls::rustls::sign::CertifiedKey;
 use tokio_rustls::rustls::{RootCertStore, ServerConfig};
 use tokio_rustls::{LazyConfigAcceptor, TlsConnector};
+use tokio_stream::wrappers::WatchStream;
+use tokio_stream::StreamExt;
 use tracing::instrument;
 use ts_rs::TS;
 
 use crate::db::model::Database;
+use crate::net::acme::AcmeCertCache;
 use crate::net::static_server::server_error;
 use crate::prelude::*;
 use crate::util::io::BackTrackingIO;
+use crate::util::sync::SyncMutex;
 use crate::util::serde::MaybeUtf8String;
 
+#[derive(Debug)]
+struct SingleCertResolver(Arc<CertifiedKey>);
+impl ResolvesServerCert for SingleCertResolver {
+    fn resolve(&self, _: tokio_rustls::rustls::server::ClientHello) -> Option<Arc<CertifiedKey>> {
+        Some(self.0.clone())
+    }
+}
+
 // not allowed: <=1024, >=32768, 5355, 5432, 9050, 6010, 9051, 5353
 
 pub struct VHostController {
+    crypto_provider: Arc<CryptoProvider>,
     db: TypedPatchDb<Database>,
     servers: Mutex<BTreeMap<u16, VHostServer>>,
 }
 impl VHostController {
     pub fn new(db: TypedPatchDb<Database>) -> Self {
         Self {
+            crypto_provider: Arc::new(tokio_rustls::rustls::crypto::ring::default_provider()),
             db,
             servers: Mutex::new(BTreeMap::new()),
         }
@@ -56,7 +73,8 @@ impl VHostController {
         let server = if let Some(server) = writable.remove(&external) {
             server
         } else {
-            VHostServer::new(external, self.db.clone()).await?
+            tracing::info!("Listening on {external}");
+            VHostServer::new(external, self.db.clone(), self.crypto_provider.clone()).await?
         };
         let rc = server
             .add(
@@ -108,7 +126,11 @@ struct VHostServer {
 }
 impl VHostServer {
     #[instrument(skip_all)]
-    async fn new(port: u16, db: TypedPatchDb<Database>) -> Result<Self, Error> {
+    async fn new(port: u16, db: TypedPatchDb<Database>, crypto_provider: Arc<CryptoProvider>) -> Result<Self, Error> {
+        let acme_tls_alpn_cache = Arc::new(SyncMutex::new(BTreeMap::<
+            InternedString,
+            watch::Receiver<Option<Arc<CertifiedKey>>>,
+        >::new()));
         // check if port allowed
         let listener = TcpListener::bind(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), port))
             .await
@@ -133,9 +155,11 @@ impl VHostServer {
                             let mut stream = BackTrackingIO::new(stream);
                             let mapping = mapping.clone();
                             let db = db.clone();
+                            let acme_tls_alpn_cache = acme_tls_alpn_cache.clone();
+                            let crypto_provider = crypto_provider.clone();
                             tokio::spawn(async move {
                                 if let Err(e) = async {
-                                    let mid = match LazyConfigAcceptor::new(
+                                    let mid: tokio_rustls::StartHandshake<&mut BackTrackingIO<TcpStream>> = match LazyConfigAcceptor::new(
                                         Acceptor::default(),
                                         &mut stream,
                                     )
@@ -206,38 +230,102 @@ impl VHostServer {
                                             .map(|(target, _)| target.clone())
                                     };
                                     if let Some(target) = target {
-                                        let mut tcp_stream =
+                                        let peek = db.peek().await;
-                                            TcpStream::connect(target.addr).await?;
+                                        let root = peek.as_private().as_key_store().as_local_certs().as_root_cert().de()?;
-                                        let hostnames = target_name
+                                        let mut cfg = match async {
-                                            .into_iter()
+                                            if let Some(acme_settings) = peek.as_public().as_server_info().as_acme().de()? {
-                                            .chain(
+                                                if let Some(domain) = target_name.as_ref().filter(|target_name| acme_settings.domains.contains(*target_name)) {
-                                                db.peek()
+                                                    if mid
-                                                    .await
+                                                        .client_hello()
-                                                    .into_public()
+                                                        .alpn()
-                                                    .into_server_info()
+                                                        .into_iter()
-                                                    .into_ip_info()
+                                                        .flatten()
-                                                    .into_entries()?
+                                                        .any(|alpn| alpn == ACME_TLS_ALPN_NAME)
-                                                    .into_iter()
+                                                    {
-                                                    .flat_map(|(_, ips)| [
+                                                        let cert = WatchStream::new(
-                                                        ips.as_ipv4().de().map(|ip| ip.map(IpAddr::V4)),
+                                                            acme_tls_alpn_cache.peek(|c| c.get(&**domain).cloned())
-                                                        ips.as_ipv6().de().map(|ip| ip.map(IpAddr::V6))
+                                                                .ok_or_else(|| {
-                                                    ])
+                                                                    Error::new(
-                                                    .filter_map(|a| a.transpose())
+                                                                        eyre!("No challenge recv available for {domain}"),
-                                                    .map(|a| a.map(|ip| InternedString::from_display(&ip)))
+                                                                        ErrorKind::OpenSsl
-                                                    .collect::<Result<Vec<_>, _>>()?,
+                                                                    )
-                                            )
+                                                                })?,
-                                            .collect();
+                                                        );
-                                        let key = db
+                                                        tracing::info!("Waiting for verification cert for {domain}");
-                                            .mutate(|v| {
+                                                        let cert = cert
-                                                v.as_private_mut()
+                                                            .filter(|c| c.is_some())
-                                                    .as_key_store_mut()
+                                                            .next()
-                                                    .as_local_certs_mut()
+                                                            .await
-                                                    .cert_for(&hostnames)
+                                                            .flatten()
-                                            })
+                                                            .ok_or_else(|| {
-                                            .await?;
+                                                                Error::new(eyre!("No challenge available for {domain}"), ErrorKind::OpenSsl)
-                                        let cfg = ServerConfig::builder()
+                                                            })?;
-                                            .with_no_client_auth();
+                                                        tracing::info!("Verification cert received for {domain}");
-                                        let mut cfg =
+                                                        let mut cfg = ServerConfig::builder_with_provider(crypto_provider.clone())
+                                                            .with_safe_default_protocol_versions()
+                                                            .with_kind(crate::ErrorKind::OpenSsl)?
+                                                            .with_no_client_auth()
+                                                            .with_cert_resolver(Arc::new(SingleCertResolver(cert)));
+
+                                                        cfg.alpn_protocols = vec![ACME_TLS_ALPN_NAME.to_vec()];
+                                                        return Ok(Err(cfg));
+                                                    } else {
+                                                        let domains = [domain.to_string()];
+                                                        let (send, recv) = watch::channel(None);
+                                                        acme_tls_alpn_cache.mutate(|c| c.insert(domain.clone(), recv));
+                                                        let cert = 
+                                                            async_acme::rustls_helper::order(
+                                                                |_, cert| {
+                                                                    send.send_replace(Some(Arc::new(cert)));
+                                                                    Ok(())
+                                                                },
+                                                                acme_settings.provider.as_str(),
+                                                                &domains,
+                                                                Some(&AcmeCertCache(&db)),
+                                                                &acme_settings.contact,
+                                                            )
+                                                            .await
+                                                            .with_kind(ErrorKind::OpenSsl)?;
+                                                        return Ok(Ok(
+                                                                ServerConfig::builder_with_provider(crypto_provider.clone())
+                                                                    .with_safe_default_protocol_versions()
+                                                                    .with_kind(crate::ErrorKind::OpenSsl)?
+                                                                    .with_no_client_auth()
+                                                                    .with_cert_resolver(Arc::new(SingleCertResolver(Arc::new(cert))))
+                                                            ));
+                                                    }
+                                                }
+                                            }
+                                            let hostnames = target_name
+                                                .into_iter()
+                                                .chain(
+                                                    peek
+                                                        .as_public()
+                                                        .as_server_info()
+                                                        .as_ip_info()
+                                                        .as_entries()?
+                                                        .into_iter()
+                                                        .flat_map(|(_, ips)| [
+                                                            ips.as_ipv4().de().map(|ip| ip.map(IpAddr::V4)),
+                                                            ips.as_ipv6().de().map(|ip| ip.map(IpAddr::V6))
+                                                        ])
+                                                        .filter_map(|a| a.transpose())
+                                                        .map(|a| a.map(|ip| InternedString::from_display(&ip)))
+                                                        .collect::<Result<Vec<_>, _>>()?,
+                                                )
+                                                .collect();
+                                            let key = db
+                                                .mutate(|v| {
+                                                    v.as_private_mut()
+                                                        .as_key_store_mut()
+                                                        .as_local_certs_mut()
+                                                        .cert_for(&hostnames)
+                                                })
+                                                .await?;
+                                            let cfg = ServerConfig::builder_with_provider(crypto_provider.clone())
+                                                .with_safe_default_protocol_versions()
+                                                .with_kind(crate::ErrorKind::OpenSsl)?
+                                                .with_no_client_auth();
                                             if mid.client_hello().signature_schemes().contains(
                                                 &tokio_rustls::rustls::SignatureScheme::ED25519,
                                             ) {
@@ -275,16 +363,34 @@ impl VHostServer {
                                                     )),
                                                 )
                                             }
-                                            .with_kind(crate::ErrorKind::OpenSsl)?;
+                                            .with_kind(crate::ErrorKind::OpenSsl)
+                                            .map(Ok)
+                                        }.await? {
+                                            Ok(a) => a,
+                                            Err(cfg) => {
+                                                tracing::info!("performing ACME auth challenge");
+                                                let mut accept = mid.into_stream(Arc::new(cfg));
+                                                let io = accept.get_mut().unwrap();
+                                                let buffered = io.stop_buffering();
+                                                io.write_all(&buffered).await?;
+                                                accept.await?;
+                                                tracing::info!("ACME auth challenge completed");
+                                                return Ok(());
+                                            }
+                                        };
+                                        let mut tcp_stream =
+                                            TcpStream::connect(target.addr).await?;
                                         match target.connect_ssl {
                                             Ok(()) => {
                                                 let mut client_cfg =
-                                                tokio_rustls::rustls::ClientConfig::builder()
+                                                tokio_rustls::rustls::ClientConfig::builder_with_provider(crypto_provider)
+                                                        .with_safe_default_protocol_versions()
+                                                        .with_kind(crate::ErrorKind::OpenSsl)?
                                                         .with_root_certificates({
                                                             let mut store = RootCertStore::empty();
                                                             store.add(
                                                         CertificateDer::from(
-                                                            key.root.to_der()?,
+                                                            root.to_der()?,
                                                         ),
                                                     ).with_kind(crate::ErrorKind::OpenSsl)?;
                                                             store

core/startos/src/service/mod.rs

@@ -16,7 +16,7 @@ use futures::stream::FusedStream;
 use futures::{SinkExt, StreamExt, TryStreamExt};
 use imbl_value::{json, InternedString};
 use itertools::Itertools;
-use models::{ActionId, ImageId, PackageId, ProcedureName};
+use models::{ActionId, HostId, ImageId, PackageId, ProcedureName};
 use nix::sys::signal::Signal;
 use persistent_container::{PersistentContainer, Subcontainer};
 use rpc_toolkit::{from_fn_async, CallRemoteHandler, Empty, HandlerArgs, HandlerFor};
@@ -603,6 +603,30 @@ impl Service {
             memory_usage: MiB::from_MiB(used),
         })
     }
+
+    pub async fn update_host(&self, host_id: HostId) -> Result<(), Error> {
+        let host = self
+            .seed
+            .ctx
+            .db
+            .peek()
+            .await
+            .as_public()
+            .as_package_data()
+            .as_idx(&self.seed.id)
+            .or_not_found(&self.seed.id)?
+            .as_hosts()
+            .as_idx(&host_id)
+            .or_not_found(&host_id)?
+            .de()?;
+        self.seed
+            .persistent_container
+            .net_service
+            .lock()
+            .await
+            .update(host_id, host)
+            .await
+    }
 }
 
 #[derive(Debug, Clone)]

core/startos/src/util/serde.rs

@@ -1029,6 +1029,12 @@ impl<T: TryFrom<Vec<u8>>> FromStr for Base64<T> {
             })
     }
 }
+impl<T: TryFrom<Vec<u8>>> ValueParserFactory for Base64<T> {
+    type Parser = FromStrParser<Self>;
+    fn value_parser() -> Self::Parser {
+        Self::Parser::new()
+    }
+}
 impl<'de, T: TryFrom<Vec<u8>>> Deserialize<'de> for Base64<T> {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
@@ -1215,6 +1221,30 @@ impl PemEncoding for ed25519_dalek::SigningKey {
     }
 }
 
+#[derive(Clone, Debug)]
+pub struct Pkcs8Doc {
+    pub tag: String,
+    pub document: pkcs8::Document,
+}
+
+impl PemEncoding for Pkcs8Doc {
+    fn from_pem<E: serde::de::Error>(pem: &str) -> Result<Self, E> {
+        let (tag, document) = pkcs8::Document::from_pem(pem).map_err(E::custom)?;
+        Ok(Pkcs8Doc {
+            tag: tag.into(),
+            document,
+        })
+    }
+    fn to_pem<E: serde::ser::Error>(&self) -> Result<String, E> {
+        der::pem::encode_string(
+            &self.tag,
+            pkcs8::LineEnding::default(),
+            self.document.as_bytes(),
+        )
+        .map_err(E::custom)
+    }
+}
+
 pub mod pem {
     use serde::{Deserialize, Deserializer, Serializer};
 

sdk/base/lib/osBindings/AcmeSettings.ts

@@ -0,0 +1,13 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type AcmeSettings = {
+  provider: string
+  /**
+   * email addresses for letsencrypt
+   */
+  contact: Array<string>
+  /**
+   * domains to get letsencrypt certs for
+   */
+  domains: string[]
+}

sdk/base/lib/osBindings/ServerInfo.ts

@@ -1,4 +1,5 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { AcmeSettings } from "./AcmeSettings"
 import type { Governor } from "./Governor"
 import type { IpInfo } from "./IpInfo"
 import type { LshwDevice } from "./LshwDevice"
@@ -22,6 +23,7 @@ export type ServerInfo = {
    */
   torAddress: string
   ipInfo: { [key: string]: IpInfo }
+  acme: AcmeSettings | null
   statusInfo: ServerStatus
   wifi: WifiInfo
   unreadNotificationCount: number

sdk/base/lib/osBindings/index.ts

@@ -1,4 +1,5 @@
 export { AcceptSigners } from "./AcceptSigners"
+export { AcmeSettings } from "./AcmeSettings"
 export { ActionId } from "./ActionId"
 export { ActionInput } from "./ActionInput"
 export { ActionMetadata } from "./ActionMetadata"

web/projects/ui/src/app/services/api/mock-patch.ts

@@ -56,6 +56,7 @@ export const mockPatchData: DataModel = {
         ipv6Range: 'FE80:CD00:0000:0CDE:1257:0000:211E:729CD/64',
       },
     },
+    acme: null,
     unreadNotificationCount: 4,
     // password is asdfasdf
     passwordHash: