Fix/integration/0.2.11 (#265)

* backports tor security fix to 0.2.10, adds functionality to allow for ssh key management during an update (#263)

* actually upgrade to 0.3.5.14-1

* update lan services on backup restore

* reload nginx, update welcome message, move reset lan to handler

* moves lan refresh after backup restore to asynchronous part of restore

* fix certificate generation

* match guards

Co-authored-by: Keagan McClelland <keagan.mcclelland@gmail.com>

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 /*.img
 /buster.zip
 /product_key

agent/config/settings.yml

@@ -33,5 +33,5 @@ database:
   database: "start9_agent.sqlite3"
   poolsize: "_env:YESOD_SQLITE_POOLSIZE:10"
 
-app-mgr-version-spec: "=0.2.10"
+app-mgr-version-spec: "=0.2.11"
 #analytics: UA-YOURCODE

agent/migrations/0.2.10::0.2.11

@@ -0,0 +1 @@
+SELECT TRUE;

agent/package.yaml

@@ -1,5 +1,5 @@
 name: ambassador-agent
-version: 0.2.10
+version: 0.2.11
 
 default-extensions:
   - NoImplicitPrelude

agent/src/Foundation.hs

@@ -186,6 +186,8 @@ cutoffDuringUpdate m = do
             path <- asks $ pathInfo . reqWaiRequest . handlerRequest
             case path of
                 [v] | v == "v" <> (show . major $ agentVersion) -> m
+                [auth] | auth == "auth" -> m
+                (_:ssh:_) | ssh == "sshKeys" -> m
                 _ -> handleS9ErrT $ throwE UpdateInProgressE
         Nothing -> m
 

agent/src/Handler/Apps.hs

@@ -109,7 +109,11 @@ type AllEffects m
                             ( Labelled
                                   "databaseConnection"
                                   (ReaderT ConnectionPool)
-                                  (ReaderT AgentCtx (ErrorC S9Error (LiftC m)))
+                                  ( Labelled
+                                        "lanThread"
+                                        (ReaderT (MVar ThreadId))
+                                        (ReaderT AgentCtx (ErrorC S9Error (LiftC m)))
+                                  )
                             )
                       )
                 )
@@ -122,6 +126,8 @@ intoHandler m = do
     runM
         . handleS9ErrC
         . flip runReaderT ctx
+        . flip runReaderT (appLanThread ctx)
+        . runLabelled @"lanThread"
         . flip runReaderT (appConnPool ctx)
         . runLabelled @"databaseConnection"
         . flip runReaderT fsbase
@@ -376,6 +382,7 @@ postUninstallAppLogic :: ( HasFilesystemBase sig m
                          , MonadIO m
                          , HasLabelled "databaseConnection" (Reader ConnectionPool) sig m
                          , HasLabelled "iconTagCache" (Reader (TVar (HM.HashMap AppId (Digest MD5)))) sig m
+                         , HasLabelled "lanThread" (Reader (MVar ThreadId)) sig m
                          )
                       => AppId
                       -> AppMgr2.DryRun
@@ -413,6 +420,7 @@ postInstallNewAppR appId = do
 
 postInstallNewAppLogic :: forall sig m a
                         . ( Has (Reader AgentCtx) sig m
+                          , HasLabelled "lanThread" (Reader (MVar ThreadId)) sig m
                           , HasLabelled "databaseConnection" (Reader ConnectionPool) sig m
                           , HasLabelled "iconTagCache" (Reader (TVar (HM.HashMap AppId (Digest MD5)))) sig m
                           , Has (Error S9Error) sig m

agent/src/Handler/Backups.hs

@@ -7,11 +7,11 @@ import           Startlude               hiding ( Reader
                                                 , runReader
                                                 )
 
-import           Control.Effect.Labelled hiding ( Handler )
-import           Control.Effect.Reader.Labelled
 import           Control.Carrier.Error.Church
 import           Control.Carrier.Lift
 import           Control.Carrier.Reader         ( runReader )
+import           Control.Effect.Labelled hiding ( Handler )
+import           Control.Effect.Reader.Labelled
 import           Data.Aeson
 import qualified Data.HashMap.Strict           as HM
 import           Data.UUID.V4
@@ -20,8 +20,13 @@ import           Yesod.Auth
 import           Yesod.Core
 import           Yesod.Core.Types
 
+import           Control.Concurrent.STM
+import           Exinst
 import           Foundation
+import           Handler.Network
 import           Handler.Util
+import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
+import           Lib.Background
 import           Lib.Error
 import qualified Lib.External.AppMgr           as AppMgr
 import qualified Lib.Notifications             as Notifications
@@ -29,10 +34,6 @@ import           Lib.Password
 import           Lib.Types.Core
 import           Lib.Types.Emver
 import           Model
-import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
-import           Lib.Background
-import           Control.Concurrent.STM
-import           Exinst
 
 
 data CreateBackupReq = CreateBackupReq
@@ -59,7 +60,8 @@ instance FromJSON RestoreBackupReq where
 
 data EjectDiskReq = EjectDiskReq
     { ejectDiskLogicalName :: Text
-    } deriving (Eq, Show)
+    }
+    deriving (Eq, Show)
 instance FromJSON EjectDiskReq where
     parseJSON = withObject "Eject Disk Req" $ \o -> do
         ejectDiskLogicalName <- o .: "logicalName"
@@ -100,6 +102,8 @@ postRestoreBackupR appId = disableEndpointOnFailedUpdate $ do
         & runReader appConnPool
         & runLabelled @"backgroundJobCache"
         & runReader appBackgroundJobs
+        & runLabelled @"lanThread"
+        & runReader appLanThread
         & handleS9ErrC
         & runM
 
@@ -173,6 +177,7 @@ stopBackupLogic appId = do
 
 restoreBackupLogic :: ( HasLabelled "backgroundJobCache" (Reader (TVar JobCache)) sig m
                       , HasLabelled "databaseConnection" (Reader ConnectionPool) sig m
+                      , HasLabelled "lanThread" (Reader (MVar ThreadId)) sig m
                       , Has (Error S9Error) sig m
                       , Has AppMgr2.AppMgr sig m
                       , MonadIO m
@@ -181,10 +186,11 @@ restoreBackupLogic :: ( HasLabelled "backgroundJobCache" (Reader (TVar JobCache)
                    -> RestoreBackupReq
                    -> m ()
 restoreBackupLogic appId RestoreBackupReq {..} = do
-    jobCache <- ask @"backgroundJobCache"
+    lanThread <- ask @"lanThread"
-    db       <- ask @"databaseConnection"
+    jobCache  <- ask @"backgroundJobCache"
-    version  <- fmap AppMgr2.infoResVersion $ AppMgr2.info [AppMgr2.flags| |] appId `orThrowM` NotFoundE "appId"
+    db        <- ask @"databaseConnection"
-                                                                                                         (show appId)
+    version   <- fmap AppMgr2.infoResVersion $ AppMgr2.info [AppMgr2.flags| |] appId `orThrowM` NotFoundE "appId"
+                                                                                                          (show appId)
     res <- liftIO . atomically $ do
         (JobCache jobs) <- readTVar jobCache
         case HM.lookup appId jobs of
@@ -206,10 +212,13 @@ restoreBackupLogic appId RestoreBackupReq {..} = do
                 let notif = case appmgrRes of
                         Left  e -> Notifications.RestoreFailed e
                         Right _ -> Notifications.RestoreSucceeded
+                resetRes <- runExceptT @S9Error $ runReader lanThread . runLabelled @"lanThread" $ postResetLanLogic
+                case resetRes of
+                    Left  _  -> pure () -- temporarily forbidden is the only possible thing here so ignore it
+                    Right () -> pure ()
                 flip runSqlPool db $ void $ Notifications.emit appId version notif
             liftIO . atomically $ modifyTVar jobCache (insertJob appId Restore tid)
 
-
 listDisksLogic :: (Has (Error S9Error) sig m, MonadIO m) => m [AppMgr.DiskInfo]
 listDisksLogic = runExceptT AppMgr.diskShow >>= liftEither
 

agent/src/Handler/Network.hs

@@ -1,16 +1,19 @@
 module Handler.Network where
 
 import           Startlude               hiding ( Reader
+                                                , ask
                                                 , asks
                                                 , runReader
                                                 )
 
 import           Control.Carrier.Lift           ( runM )
 import           Control.Effect.Error
-import           Control.Carrier.Reader
 import           Lib.Error
 import           Yesod.Core                     ( getYesod )
 
+import           Control.Carrier.Reader         ( runReader )
+import           Control.Effect.Labelled        ( runLabelled )
+import           Control.Effect.Reader.Labelled
 import           Foundation
 import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
 import           Lib.Types.Core
@@ -18,11 +21,12 @@ import           Lib.Types.Core
 postResetLanR :: Handler ()
 postResetLanR = do
     ctx <- getYesod
-    runM . handleS9ErrC . runReader ctx $ postResetLanLogic
+    runM . handleS9ErrC . runReader (appLanThread ctx) . runLabelled @"lanThread" $ postResetLanLogic
 
-postResetLanLogic :: (MonadIO m, Has (Reader AgentCtx) sig m, Has (Error S9Error) sig m) => m ()
+postResetLanLogic :: (MonadIO m, HasLabelled "lanThread" (Reader (MVar ThreadId)) sig m, Has (Error S9Error) sig m)
+                  => m ()
 postResetLanLogic = do
-    threadVar <- asks appLanThread
+    threadVar <- ask @"lanThread"
     mtid      <- liftIO . tryTakeMVar $ threadVar
     case mtid of
         Nothing  -> throwError $ TemporarilyForbiddenE (AppId "LAN") "reset" "being reset"

agent/src/Lib/Algebra/Domain/AppMgr.hs

@@ -11,8 +11,7 @@ module Lib.Algebra.Domain.AppMgr
     ( module Lib.Algebra.Domain.AppMgr
     , module Lib.Algebra.Domain.AppMgr.Types
     , module Lib.Algebra.Domain.AppMgr.TH
-    )
+    ) where
-where
 
 import           Startlude
 
@@ -26,31 +25,31 @@ import           Data.Singletons.Prelude hiding ( Error )
 import           Data.Singletons.Prelude.Either
 import qualified Data.String                   as String
 
-import           Lib.Algebra.Domain.AppMgr.Types
+import           Control.Monad.Base             ( MonadBase(..) )
+import           Control.Monad.Fail             ( MonadFail(fail) )
+import           Control.Monad.Trans.Class      ( MonadTrans )
+import           Control.Monad.Trans.Control    ( MonadBaseControl(..)
+                                                , MonadTransControl(..)
+                                                , defaultLiftBaseWith
+                                                , defaultRestoreM
+                                                )
+import           Control.Monad.Trans.Resource   ( MonadResource(..) )
+import qualified Data.ByteString.Char8         as C8
+import qualified Data.ByteString.Lazy          as LBS
+import           Data.String.Interpolate.IsString
+                                                ( i )
 import           Lib.Algebra.Domain.AppMgr.TH
+import           Lib.Algebra.Domain.AppMgr.Types
 import           Lib.Error
 import qualified Lib.External.AppManifest      as Manifest
 import           Lib.TyFam.ConditionalData
-import           Lib.Types.Core                 ( AppId(..)
+import           Lib.Types.Core                 ( AppContainerStatus(..)
-                                                , AppContainerStatus(..)
+                                                , AppId(..)
                                                 )
-import           Lib.Types.NetAddress
 import           Lib.Types.Emver
-import           Control.Monad.Trans.Class      ( MonadTrans )
+import           Lib.Types.NetAddress
-import qualified Data.ByteString.Lazy          as LBS
-import           System.Process.Typed
-import           Data.String.Interpolate.IsString
-                                                ( i )
-import           Control.Monad.Base             ( MonadBase(..) )
-import           Control.Monad.Fail             ( MonadFail(fail) )
-import           Control.Monad.Trans.Resource   ( MonadResource(..) )
-import           Control.Monad.Trans.Control    ( defaultLiftBaseWith
-                                                , defaultRestoreM
-                                                , MonadTransControl(..)
-                                                , MonadBaseControl(..)
-                                                )
-import qualified Data.ByteString.Char8         as C8
 import           System.Process
+import           System.Process.Typed
 
 
 type InfoRes :: Either OnlyInfoFlag [IncludeInfoFlag] -> Type
@@ -371,13 +370,16 @@ instance (Has (Error S9Error) sig m, Algebra sig m, MonadIO m) => Algebra (AppMg
         (L (List (SRight flags))) -> do
             let renderedFlags = (genInclusiveFlag <$> fromSing flags) <> ["--json"]
             let args          = "list" : renderedFlags
-            (ec, out) <- readProcessInheritStderr "appmgr" args ""
+            let runIt retryCount = do
-            res       <- case ec of
+                    (ec, out) <- readProcessInheritStderr "appmgr" args ""
-                ExitSuccess -> case withSingI flags $ eitherDecodeStrict out of
+                    case ec of
-                    Left  e -> throwError $ AppMgrParseE (toS $ String.unwords args) (decodeUtf8 out) e
+                        ExitSuccess -> case withSingI flags $ eitherDecodeStrict out of
-                    Right x -> pure x
+                            Left  e -> throwError $ AppMgrParseE (toS $ String.unwords args) (decodeUtf8 out) e
-                ExitFailure n -> throwError $ AppMgrE "list" n
+                            Right x -> pure $ ctx $> x
-            pure $ ctx $> res
+                        ExitFailure 6 ->
+                            if retryCount > 0 then runIt (retryCount - 1) else throwError $ AppMgrE "list" 6
+                        ExitFailure n -> throwError $ AppMgrE "list" n
+            runIt (1 :: Word) -- with 1 retry
         (L (Remove dryorpurge appId)) -> do
             let args = "remove" : case dryorpurge of
                     Left  (DryRun True) -> ["--dry-run", show appId, "--json"]

agent/src/Lib/Synchronizers.hs

@@ -48,6 +48,7 @@ import           System.Process                 ( callCommand )
 
 import           Constants
 import           Control.Effect.Error    hiding ( run )
+import           Control.Effect.Labelled        ( runLabelled )
 import           Daemon.ZeroConf                ( getStart9AgentHostname )
 import qualified Data.Text                     as T
 import           Foundation
@@ -97,12 +98,12 @@ parseKernelVersion = do
     pure $ KernelVersion (Version (major', minor', patch', 0)) arch
 
 synchronizer :: Synchronizer
-synchronizer = sync_0_2_10
+synchronizer = sync_0_2_11
 {-# INLINE synchronizer #-}
 
-sync_0_2_10 :: Synchronizer
+sync_0_2_11 :: Synchronizer
-sync_0_2_10 = Synchronizer
+sync_0_2_11 = Synchronizer
-    "0.2.10"
+    "0.2.11"
     [ syncCreateAgentTmp
     , syncCreateSshDir
     , syncRemoveAvahiSystemdDependency
@@ -126,6 +127,7 @@ sync_0_2_10 = Synchronizer
     , syncRestarterService
     , syncInstallEject
     , syncDropCertificateUniqueness
+    , syncRemoveDefaultNginxCfg
     ]
 
 syncCreateAgentTmp :: SyncOp
@@ -437,10 +439,11 @@ syncInstallAppMgr = SyncOp "Install AppMgr" check migrate False
             Left  _ -> pure True
             Right v -> not . (v <||) <$> asks (appMgrVersionSpec . appSettings)
         migrate = fmap (either absurd id) . runExceptT . flip catchE failUpdate $ do
+            lan <- asks appLanThread
             avs <- asks $ appMgrVersionSpec . appSettings
             av  <- AppMgr.installNewAppMgr avs
             unless (av <|| avs) $ throwE $ AppMgrVersionE av avs
-            postResetLanLogic -- to accommodate 0.2.x -> 0.2.9 where previous appmgr didn't correctly set up lan
+            flip runReaderT lan $ runLabelled @"lanThread" $ postResetLanLogic -- to accommodate 0.2.x -> 0.2.9 where previous appmgr didn't correctly set up lan
 
 syncUpgradeLifeline :: SyncOp
 syncUpgradeLifeline = SyncOp "Upgrade Lifeline" check migrate False
@@ -582,11 +585,11 @@ syncRestarterService = SyncOp "Install Restarter Service" check migrate True
             liftIO $ callCommand "systemctl enable restarter.timer"
 
 syncUpgradeTor :: SyncOp
-syncUpgradeTor = SyncOp "Install Tor 0.3.5.12-1" check migrate False
+syncUpgradeTor = SyncOp "Install Tor 0.3.5.14-1" check migrate False
     where
         check =
             liftIO
-                $       (  run (shell [i|dpkg -l|] $| shell [i|grep tor|] $| shell [i|grep 0.3.5.12-1|] $| conduit await)
+                $       (  run (shell [i|dpkg -l|] $| shell [i|grep tor|] $| shell [i|grep 0.3.5.14-1|] $| conduit await)
                         $> False
                         )
                 `catch` \(e :: ProcessException) -> case e of
@@ -594,7 +597,7 @@ syncUpgradeTor = SyncOp "Install Tor 0.3.5.12-1" check migrate False
                             _ -> throwIO e
         migrate = liftIO . run $ do
             shell "apt-get update"
-            shell "apt-get install -y tor=0.3.5.12-1"
+            shell "apt-get install -y tor=0.3.5.14-1"
 
 syncDropCertificateUniqueness :: SyncOp
 syncDropCertificateUniqueness = SyncOp "Eliminate OpenSSL unique_subject=yes" check migrate False
@@ -602,14 +605,33 @@ syncDropCertificateUniqueness = SyncOp "Eliminate OpenSSL unique_subject=yes" ch
         uni   = "unique_subject = no\n"
         check = do
             base         <- asks $ appFilesystemBase . appSettings
-            contentsRoot <- liftIO . BS.readFile . toS $ (rootCaDirectory <> "index.txt.attr") `relativeTo` base
+            contentsRoot <-
-            contentsInt  <- liftIO . BS.readFile . toS $ (intermediateCaDirectory <> "index.txt.attr") `relativeTo` base
+                liftIO
-            pure $ uni /= contentsRoot || uni /= contentsInt
+                $       (fmap Just . BS.readFile . toS $ (rootCaDirectory <> "index.txt.attr") `relativeTo` base)
+                `catch` \(e :: IOException) -> if isDoesNotExistError e then pure Nothing else throwIO e
+            contentsInt <-
+                liftIO
+                $ (fmap Just . BS.readFile . toS $ (intermediateCaDirectory <> "index.txt.attr") `relativeTo` base)
+                `catch` \(e :: IOException) -> if isDoesNotExistError e then pure Nothing else throwIO e
+            case (contentsRoot, contentsInt) of
+                (Just root, Just int) -> pure $ uni /= root || uni /= int
+                _                     -> pure True
         migrate = do
             base <- asks $ appFilesystemBase . appSettings
             liftIO $ BS.writeFile (toS $ (rootCaDirectory <> "index.txt.attr") `relativeTo` base) uni
             liftIO $ BS.writeFile (toS $ (intermediateCaDirectory <> "index.txt.attr") `relativeTo` base) uni
 
+syncRemoveDefaultNginxCfg :: SyncOp
+syncRemoveDefaultNginxCfg = SyncOp "Remove Default Nginx Configuration" check migrate False
+    where
+        check = do
+            base <- asks $ appFilesystemBase . appSettings
+            liftIO $ doesPathExist (toS $ nginxSitesEnabled "default" `relativeTo` base)
+        migrate = do
+            base <- asks $ appFilesystemBase . appSettings
+            liftIO $ removeFileIfExists (toS $ nginxSitesEnabled "default" `relativeTo` base)
+            liftIO $ systemCtl RestartService "nginx" $> ()
+
 failUpdate :: S9Error -> ExceptT Void (ReaderT AgentCtx IO) ()
 failUpdate e = do
     ref <- asks appIsUpdateFailed

appmgr/.gitignore

@@ -1,2 +1,3 @@
 /target
 **/*.rs.bk
+.DS_Store

appmgr/Cargo.lock

@@ -41,7 +41,7 @@ checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1"
 
 [[package]]
 name = "appmgr"
-version = "0.2.10"
+version = "0.2.11"
 dependencies = [
  "async-trait",
  "avahi-sys",

appmgr/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Aiden McClelland <me@drbonez.dev>"]
 edition = "2018"
 name = "appmgr"
-version = "0.2.10"
+version = "0.2.11"
 
 [lib]
 name = "appmgrlib"

appmgr/build-dev.sh

@@ -3,6 +3,11 @@
 set -e
 shopt -s expand_aliases
 
+if [ "$0" != "./build-dev.sh" ]; then
+	>&2 echo "Must be run from appmgr directory"
+	exit 1
+fi
+
 alias 'rust-arm-builder'='docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:latest'
 
 cd ..

appmgr/build-portable.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+shopt -s expand_aliases
+
+if [ "$0" != "./build-portable.sh" ]; then
+	>&2 echo "Must be run from appmgr directory"
+	exit 1
+fi
+
+alias 'rust-musl-builder'='docker run --rm -it -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src messense/rust-musl-cross:x86_64-musl'
+
+cd ..
+rust-musl-builder sh -c "(cd appmgr && cargo build --release --target=x86_64-unknown-linux-musl --features=portable,production --no-default-features)"
+cd appmgr

appmgr/build-prod.sh

@@ -3,6 +3,11 @@
 set -e
 shopt -s expand_aliases
 
+if [ "$0" != "./build-prod.sh" ]; then
+	>&2 echo "Must be run from appmgr directory"
+	exit 1
+fi
+
 alias 'rust-arm-builder'='docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:latest'
 
 cd ..

appmgr/src/backup.rs

@@ -1,3 +1,4 @@
+use std::os::unix::process::ExitStatusExt;
 use std::path::Path;
 
 use argon2::Config;
@@ -10,6 +11,7 @@ use serde::Serialize;
 use crate::util::from_yaml_async_reader;
 use crate::util::to_yaml_async_writer;
 use crate::util::Invoke;
+use crate::util::PersistencePath;
 use crate::version::VersionT;
 use crate::Error;
 use crate::ResultExt;
@@ -224,6 +226,28 @@ pub async fn restore_backup<P: AsRef<Path>>(
     }
 
     crate::tor::restart().await?;
+    // Delete the fullchain certificate, so it can be regenerated with the restored tor pubkey address
+    PersistencePath::from_ref("apps")
+        .join(&app_id)
+        .join("cert-local.fullchain.crt.pem")
+        .delete()
+        .await?;
+    crate::tor::write_lan_services(
+        &crate::tor::services_map(&PersistencePath::from_ref(crate::SERVICES_YAML)).await?,
+    )
+    .await?;
+    let svc_exit = std::process::Command::new("service")
+        .args(&["nginx", "reload"])
+        .status()?;
+    crate::ensure_code!(
+        svc_exit.success(),
+        crate::error::GENERAL_ERROR,
+        "Failed to Reload Nginx: {}",
+        svc_exit
+            .code()
+            .or_else(|| { svc_exit.signal().map(|a| 128 + a) })
+            .unwrap_or(0)
+    );
 
     Ok(())
 }

appmgr/src/control.rs

@@ -24,7 +24,6 @@ pub async fn start_app(name: &str, update_metadata: bool) -> Result<(), Error> {
     if status == crate::apps::DockerStatus::Stopped {
         if update_metadata {
             crate::config::configure(name, None, None, false).await?;
-            crate::dependencies::update_shared(name).await?;
             crate::dependencies::update_binds(name).await?;
         }
         crate::apps::set_needs_restart(name, false).await?;

appmgr/src/dependencies.rs

@@ -188,31 +188,6 @@ pub async fn auto_configure(
     crate::config::configure(dependency, Some(dependency_config), None, dry_run).await
 }
 
-pub async fn update_shared(dependency_id: &str) -> Result<(), Error> {
-    let dependency_manifest = crate::apps::manifest(dependency_id).await?;
-    if let Some(shared) = dependency_manifest.shared {
-        for dependent_id in &crate::apps::dependents(dependency_id, false).await? {
-            let dependent_manifest = crate::apps::manifest(&dependent_id).await?;
-            if dependent_manifest
-                .dependencies
-                .0
-                .get(dependency_id)
-                .ok_or_else(|| failure::format_err!("failed to index dependent: {}", dependent_id))?
-                .mount_shared
-            {
-                tokio::fs::create_dir_all(
-                    Path::new(crate::VOLUMES)
-                        .join(dependency_id)
-                        .join(&shared)
-                        .join(&dependent_id),
-                )
-                .await?;
-            }
-        }
-    }
-    Ok(())
-}
-
 pub async fn update_binds(dependent_id: &str) -> Result<(), Error> {
     let dependent_manifest = crate::apps::manifest(dependent_id).await?;
     let dependency_manifests = futures::future::try_join_all(
@@ -222,12 +197,19 @@ pub async fn update_binds(dependent_id: &str) -> Result<(), Error> {
             .into_iter()
             .filter(|(_, info)| info.mount_public || info.mount_shared)
             .map(|(id, info)| async {
-                crate::apps::manifest(&id).await.map(|man| (id, info, man))
+                Ok::<_, Error>(if crate::apps::list_info().await?.contains_key(&id) {
+                    let man = crate::apps::manifest(&id).await?;
+                    Some((id, info, man))
+                } else {
+                    None
+                })
             }),
     )
     .await?;
     // i just have a gut feeling this shouldn't be concurrent
-    for (dependency_id, info, dependency_manifest) in dependency_manifests {
+    for (dependency_id, info, dependency_manifest) in
+        dependency_manifests.into_iter().filter_map(|a| a)
+    {
         match (dependency_manifest.public, info.mount_public) {
             (Some(public), true) => {
                 let public_path = Path::new(crate::VOLUMES).join(&dependency_id).join(public);

appmgr/src/disks.rs

@@ -1,10 +1,11 @@
 use std::path::Path;
 
+use failure::ResultExt as _;
 use futures::future::try_join_all;
 
 use crate::util::Invoke;
 use crate::Error;
-use crate::ResultExt;
+use crate::ResultExt as _;
 
 pub const FSTAB: &'static str = "/etc/fstab";
 
@@ -153,6 +154,11 @@ pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(
     dst: P1,
     read_only: bool,
 ) -> Result<(), Error> {
+    log::info!(
+        "Binding {} to {}",
+        src.as_ref().display(),
+        dst.as_ref().display()
+    );
     let is_mountpoint = tokio::process::Command::new("mountpoint")
         .arg(dst.as_ref())
         .stdout(std::process::Stdio::null())
@@ -185,6 +191,7 @@ pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(
 }
 
 pub async fn unmount<P: AsRef<Path>>(mount_point: P) -> Result<(), Error> {
+    log::info!("Unmounting {}.", mount_point.as_ref().display());
     let umount_output = tokio::process::Command::new("umount")
         .arg(mount_point.as_ref())
         .output()
@@ -192,10 +199,14 @@ pub async fn unmount<P: AsRef<Path>>(mount_point: P) -> Result<(), Error> {
     crate::ensure_code!(
         umount_output.status.success(),
         crate::error::FILESYSTEM_ERROR,
-        "Error Unmounting Drive: {}",
+        "Error Unmounting Drive: {}: {}",
+        mount_point.as_ref().display(),
         std::str::from_utf8(&umount_output.stderr).unwrap_or("Unknown Error")
     );
-    tokio::fs::remove_dir_all(mount_point.as_ref()).await?;
+    tokio::fs::remove_dir_all(mount_point.as_ref())
+        .await
+        .with_context(|e| format!("rm {}: {}", mount_point.as_ref().display(), e))
+        .with_code(crate::error::FILESYSTEM_ERROR)?;
     Ok(())
 }
 

appmgr/src/install.rs

@@ -561,14 +561,17 @@ pub async fn install_v0<R: AsyncRead + Unpin + Send + Sync>(
             crate::config::configure(&manifest.id, Some(empty_config), None, false).await?;
         }
     }
+    crate::dependencies::update_binds(&manifest.id).await?;
     for (dep_id, dep_info) in manifest.dependencies.0 {
         if dep_info.mount_shared
             && crate::apps::list_info().await?.get(&dep_id).is_some()
             && crate::apps::manifest(&dep_id).await?.shared.is_some()
-            && crate::apps::status(&dep_id, false).await?.status
-                != crate::apps::DockerStatus::Stopped
         {
-            crate::apps::set_needs_restart(&dep_id, true).await?;
+            match crate::apps::status(&dep_id, false).await?.status {
+                crate::apps::DockerStatus::Stopped => (),
+                crate::apps::DockerStatus::Running => crate::control::restart_app(&dep_id).await?,
+                _ => crate::apps::set_needs_restart(&dep_id, true).await?,
+            }
         }
     }
 

appmgr/src/nginx-standard.conf.template

@@ -9,6 +9,7 @@ server {{
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
+        client_max_body_size 0;
     }}
 }}
 server {{

appmgr/src/nginx.conf.template

@@ -4,5 +4,6 @@ server {{
     location / {{
         proxy_pass http://{app_ip}:{internal_port}/;
         proxy_set_header Host $host;
+        client_max_body_size 0;
     }}
 }}

appmgr/src/remove.rs

@@ -1,9 +1,11 @@
+use crate::failure::ResultExt;
 use std::path::Path;
 
 use linear_map::LinearMap;
 
 use crate::dependencies::{DependencyError, TaggedDependencyError};
 use crate::Error;
+use crate::ResultExt as _;
 
 pub async fn remove(
     name: &str,
@@ -55,48 +57,79 @@ pub async fn remove(
         log::info!("Removing tor hidden service.");
         crate::tor::rm_svc(name).await?;
         log::info!("Removing app metadata.");
-        tokio::fs::remove_dir_all(Path::new(crate::PERSISTENCE_DIR).join("apps").join(name))
+        let metadata_path = Path::new(crate::PERSISTENCE_DIR).join("apps").join(name);
-            .await?;
+        tokio::fs::remove_dir_all(&metadata_path)
-        log::info!("Destroying mounted volume.");
+            .await
+            .with_context(|e| format!("rm {}: {}", metadata_path.display(), e))
+            .with_code(crate::error::FILESYSTEM_ERROR)?;
         log::info!("Unbinding shared filesystem.");
-        for (dep, info) in manifest.dependencies.0.iter() {
+        let installed_apps = crate::apps::list_info().await?;
-            if info.mount_public {
+        for (dep, _) in manifest.dependencies.0.iter() {
-                crate::disks::unmount(
+            let path = Path::new(crate::VOLUMES)
-                    Path::new(crate::VOLUMES)
+                .join(name)
-                        .join(name)
+                .join("start9")
-                        .join("start9")
+                .join("public")
-                        .join("public")
+                .join(&dep);
-                        .join(&dep),
+            if path.exists() {
-                )
+                crate::disks::unmount(&path).await?;
-                .await?;
+            } else {
+                log::warn!("{} does not exist, skipping...", path.display());
             }
-            if info.mount_shared {
+            let path = Path::new(crate::VOLUMES)
-                if let Some(shared) = match crate::apps::manifest(dep).await {
+                .join(name)
-                    Ok(man) => man.shared,
+                .join("start9")
-                    Err(e) => {
+                .join("shared")
-                        log::error!("Failed to Fetch Dependency Manifest: {}", e);
+                .join(&dep);
-                        None
+            if path.exists() {
-                    }
+                crate::disks::unmount(&path).await?;
-                } {
+            } else {
-                    let path = Path::new(crate::VOLUMES)
+                log::warn!("{} does not exist, skipping...", path.display());
-                        .join(name)
+            }
-                        .join("start9")
+            if installed_apps.contains_key(dep) {
-                        .join("shared")
+                let dep_man = crate::apps::manifest(dep).await?;
-                        .join(&dep);
+                if let Some(shared) = dep_man.shared {
-                    if path.exists() {
-                        crate::disks::unmount(&path).await?;
-                    }
                     let path = Path::new(crate::VOLUMES).join(dep).join(&shared).join(name);
                     if path.exists() {
-                        tokio::fs::remove_dir_all(
+                        tokio::fs::remove_dir_all(&path)
-                            Path::new(crate::VOLUMES).join(dep).join(&shared).join(name),
+                            .await
-                        )
+                            .with_context(|e| format!("rm {}: {}", path.display(), e))
-                        .await?;
+                            .with_code(crate::error::FILESYSTEM_ERROR)?;
                     }
                 }
+            } else {
+                log::warn!("{} is not installed, skipping...", dep);
+            }
+        }
+        if manifest.public.is_some() || manifest.shared.is_some() {
+            for dependent in crate::apps::dependents(name, false).await? {
+                let path = Path::new(crate::VOLUMES)
+                    .join(&dependent)
+                    .join("start9")
+                    .join("public")
+                    .join(name);
+                if path.exists() {
+                    crate::disks::unmount(&path).await?;
+                } else {
+                    log::warn!("{} does not exist, skipping...", path.display());
+                }
+                let path = Path::new(crate::VOLUMES)
+                    .join(dependent)
+                    .join("start9")
+                    .join("shared")
+                    .join(name);
+                if path.exists() {
+                    crate::disks::unmount(&path).await?;
+                } else {
+                    log::warn!("{} does not exist, skipping...", path.display());
+                }
             }
         }
-        tokio::fs::remove_dir_all(Path::new(crate::VOLUMES).join(name)).await?;
+        log::info!("Destroying mounted volume.");
+        let volume_path = Path::new(crate::VOLUMES).join(name);
+        tokio::fs::remove_dir_all(&volume_path)
+            .await
+            .with_context(|e| format!("rm {}: {}", volume_path.display(), e))
+            .with_code(crate::error::FILESYSTEM_ERROR)?;
         log::info!("Pruning unused docker images.");
         crate::ensure_code!(
             std::process::Command::new("docker")

appmgr/src/util.rs

@@ -110,6 +110,14 @@ impl PersistencePath {
     pub async fn for_update(self) -> Result<UpdateHandle<ForRead>, Error> {
         UpdateHandle::new(self).await
     }
+
+    pub async fn delete(&self) -> Result<(), Error> {
+        match tokio::fs::remove_file(self.path()).await {
+            Ok(()) => Ok(()),
+            Err(k) if k.kind() == std::io::ErrorKind::NotFound => Ok(()),
+            e => e.with_code(crate::error::FILESYSTEM_ERROR),
+        }
+    }
 }
 
 #[derive(Debug)]

appmgr/src/version/mod.rs

@@ -17,7 +17,6 @@ mod v0_1_4;
 mod v0_1_5;
 mod v0_2_0;
 mod v0_2_1;
-mod v0_2_10;
 mod v0_2_2;
 mod v0_2_3;
 mod v0_2_4;
@@ -27,7 +26,10 @@ mod v0_2_7;
 mod v0_2_8;
 mod v0_2_9;
 
-pub use v0_2_10::Version as Current;
+mod v0_2_10;
+mod v0_2_11;
+
+pub use v0_2_11::Version as Current;
 
 #[derive(serde::Serialize, serde::Deserialize)]
 #[serde(untagged)]
@@ -50,6 +52,7 @@ enum Version {
     V0_2_8(Wrapper<v0_2_8::Version>),
     V0_2_9(Wrapper<v0_2_9::Version>),
     V0_2_10(Wrapper<v0_2_10::Version>),
+    V0_2_11(Wrapper<v0_2_11::Version>),
     Other(emver::Version),
 }
 
@@ -162,6 +165,7 @@ pub async fn init() -> Result<(), failure::Error> {
             Version::V0_2_8(v) => v.0.migrate_to(&Current::new()).await?,
             Version::V0_2_9(v) => v.0.migrate_to(&Current::new()).await?,
             Version::V0_2_10(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_11(v) => v.0.migrate_to(&Current::new()).await?,
             Version::Other(_) => (),
             // TODO find some way to automate this?
         }
@@ -253,6 +257,7 @@ pub async fn self_update(requirement: emver::VersionRange) -> Result<(), Error>
         Version::V0_2_8(v) => Current::new().migrate_to(&v.0).await?,
         Version::V0_2_9(v) => Current::new().migrate_to(&v.0).await?,
         Version::V0_2_10(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_11(v) => Current::new().migrate_to(&v.0).await?,
         Version::Other(_) => (),
         // TODO find some way to automate this?
     };

appmgr/src/version/v0_2_11.rs

@@ -0,0 +1,38 @@
+use super::*;
+use std::os::unix::process::ExitStatusExt;
+
+const V0_2_11: emver::Version = emver::Version::new(0, 2, 11, 0);
+
+pub struct Version;
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_2_10::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> &'static emver::Version {
+        &V0_2_11
+    }
+    async fn up(&self) -> Result<(), Error> {
+        crate::tor::write_lan_services(
+            &crate::tor::services_map(&PersistencePath::from_ref(crate::SERVICES_YAML)).await?,
+        )
+        .await?;
+        let svc_exit = std::process::Command::new("service")
+            .args(&["nginx", "reload"])
+            .status()?;
+        crate::ensure_code!(
+            svc_exit.success(),
+            crate::error::GENERAL_ERROR,
+            "Failed to Reload Nginx: {}",
+            svc_exit
+                .code()
+                .or_else(|| { svc_exit.signal().map(|a| 128 + a) })
+                .unwrap_or(0)
+        );
+        Ok(())
+    }
+    async fn down(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}

ui/client-manifest.yaml

@@ -1,6 +1,6 @@
 manifest-version: 0
 app-id: start9-ambassador
-app-version: 0.2.10
+app-version: 0.2.11
 uri-rewrites:
     - =/api -> http://{{start9-ambassador}}:5959/authenticate
     - /api/ -> http://{{start9-ambassador}}:5959/

ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "embassy-ui",
-  "version": "0.2.10",
+  "version": "0.2.11",
   "description": "GUI for EmbassyOS",
   "author": "Start9 Labs",
   "homepage": "https://github.com/Start9Labs/embassy-ui",

ui/src/app/app-config/config-cursor.ts

@@ -298,11 +298,11 @@ export class ConfigCursor<T extends ValueType> {
     const mappedCfg = this.mappedConfig()
     if (cfg && mappedCfg && typeof cfg === 'object' && typeof mappedCfg === 'object') {
       const spec = this.spec()
-      let allKeys
+      let allKeys: Set<string>
       if (spec.type === 'union') {
         let unionSpec = spec as  ValueSpecOf<'union'>
         const labelForSelection = unionSpec.tag.id
-        allKeys = new Set([...Object.keys(unionSpec.variants[cfg[labelForSelection]])])
+        allKeys = new Set([labelForSelection, ...Object.keys(unionSpec.variants[cfg[labelForSelection]])])
       } else {
         allKeys = new Set([...Object.keys(cfg), ...Object.keys(mappedCfg)])
       }

ui/src/app/modals/app-config-union/app-config-union.page.html

@@ -17,8 +17,15 @@
   <ion-item-group>
     <ion-item-divider></ion-item-divider>
     <ion-item>
+      <ion-icon size="small" slot="start" *ngIf="!edited"
+        style="margin-right: 15px; color: rgba(0,0,0,0); background: radial-gradient(#2a4e8970, #2a4e8970 35%, transparent 35%, transparent);"
+        name="ellipse"></ion-icon>
+      <ion-icon size="small" slot="start" *ngIf="edited" style="margin-right: 15px" color="primary" name="ellipse">
+      </ion-icon>
       <ion-label>{{ spec.tag.name }}</ion-label>
-      <ion-select slot="end" [interfaceOptions]="setSelectOptions()" placeholder="Select One" [(ngModel)]="value[spec.tag.id]" [selectedText]="spec.tag.variantNames[value[spec.tag.id]]" (ngModelChange)="handleUnionChange()">
+      <ion-select slot="end" [interfaceOptions]="setSelectOptions()" placeholder="Select One"
+        [(ngModel)]="value[spec.tag.id]" [selectedText]="spec.tag.variantNames[value[spec.tag.id]]"
+        (ngModelChange)="handleUnionChange()">
         <ion-select-option *ngFor="let option of spec.variants | keyvalue: asIsOrder" [value]="option.key">
           {{ spec.tag.variantNames[option.key] }}
           <span *ngIf="option.key === spec.default"> (default)</span>

ui/src/app/modals/app-config-union/app-config-union.page.ts

@@ -19,6 +19,7 @@ export class AppConfigUnionPage {
   spec: ValueSpecUnion
   value: object
   error: string
+  edited: boolean
 
   constructor (
     private readonly modalCtrl: ModalController,
@@ -28,6 +29,7 @@ export class AppConfigUnionPage {
     this.spec = this.cursor.spec()
     this.value = this.cursor.config()
     this.error = this.cursor.checkInvalid()
+    this.edited = this.cursor.seekNext(this.spec.tag.id).isEdited()
   }
 
   async dismiss () {
@@ -37,6 +39,8 @@ export class AppConfigUnionPage {
   async handleUnionChange () {
     this.value = mapUnionSpec(this.spec, this.value)
     this.objectConfig.annotations = this.objectConfig.cursor.getAnnotations()
+    this.error = this.cursor.checkInvalid()
+    this.edited = this.cursor.seekNext(this.spec.tag.id).isEdited()
   }
 
   setSelectOptions () {

ui/src/app/modals/os-welcome/os-welcome.page.html

@@ -1,7 +1,7 @@
 <ion-header>
   <ion-toolbar>
     <ion-title >
-      <ion-label style="font-size: 20px;" class="ion-text-wrap">Welcome to 0.2.10!</ion-label>
+      <ion-label style="font-size: 20px;" class="ion-text-wrap">Welcome to 0.2.11!</ion-label>
     </ion-title>
   </ion-toolbar>
 </ion-header>
@@ -9,14 +9,18 @@
 <ion-content class="ion-padding">
   <div style="display: flex; flex-direction: column; justify-content: space-between; height: 100%">
     <h2>Highlights</h2>
-    <p class="main-content">
+    <div class="main-content">
-      0.2.10 introduces LAN support for services running on the Embassy. A service's LAN address (.local URL) can be accessed while connected to the same network.
+      <p>This release includes several bugfixes to resolve:</p>
-      This is useful for two reasons: (1) LAN connections are significantly faster than Tor, and (2) if the Tor network is experiencing connectivity issues, you will not be locked out of your services.
+      <ol>
-
+        <li>Refreshing error messages during configuration changes</li>
-      It also introduces support for services to define one-time actions that are exposed to the user. This
+        <li>Starting services with uninstalled optional dependencies</li>
-      can be useful for password resets or other types of operations where doing it through the service UI would be
+        <li>Uninstalling services with optional dependencies</li>
-      insecure or otherwise undesirable.
+        <li>Redirecting to HTTPS when navigating to LAN address</li>
-    </p>
+        <li>Displaying warning messages during concurrent upgrades of dependent services</li>
+        <li>Allowing larger file uploads</li>
+        <li>Patching a security fix for Tor</li>
+      </ol>
+    </div>
 
     <div class="close-button">
       <ion-button fill="outline" (click)="dismiss()">

ui/src/app/services/api/mock-api.service.ts

@@ -492,8 +492,8 @@ const mockApiNotifications: ReqRes.GetNotificationsRes = [
 const mockApiServer: () => ReqRes.GetServerRes = () => ({
   serverId: 'start9-mockxyzab',
   name: 'Embassy:12345678',
-  versionInstalled: '0.2.10',
+  versionInstalled: '0.2.11',
-  versionLatest: '0.2.11',
+  versionLatest: '0.2.12',
   status: ServerStatus.RUNNING,
   alternativeRegistryUrl: 'beta-registry.start9labs.com',
   welcomeAck: true,

ui/use-mocks.json

@@ -1,5 +1,4 @@
 {
   "useMocks": false,
-  "mockOver": "tor",
   "skipStartupAlerts": false
 }