add system images to os build process (#620)

* update compat build and add to os build process

* copy to explicit file

* fix paths and loading input

* temp save docker images output

* add docker config file to suppress warning

* notes and first attempt of load image script

* pr fixes

* run as root, fix executable path

* wip fixes

* fix pool name and stop docker before umount

* start docker again

* reset docker fs

* remove mkdir

* load system images during embassy-init

* add utils to make build

* fix utils source

* create system-images dir as root

* cleanup

* make loading docker images datadir agnostic

* address PR feedback

* rework load images

* create shutdown channel on failed embassy-init run

* pr feedback

* fix import

Makefile

@@ -1,7 +1,8 @@
 EMBASSY_BINS := appmgr/target/aarch64-unknown-linux-gnu/release/embassyd appmgr/target/aarch64-unknown-linux-gnu/release/embassy-init appmgr/target/aarch64-unknown-linux-gnu/release/embassy-cli appmgr/target/aarch64-unknown-linux-gnu/release/embassy-sdk
 EMBASSY_UIS := ui/www setup-wizard/www diagnostic-ui/www
 EMBASSY_SRC := ubuntu.img product_key.txt $(EMBASSY_BINS) appmgr/embassyd.service appmgr/embassy-init.service $(EMBASSY_UIS) $(shell find build)
-
+COMPAT_SRC := $(shell find system-images/compat/src)
+UTILS_SRC := $(shell find system-images/utils/Dockerfile)
 APPMGR_SRC := $(shell find appmgr/src) $(shell find patch-db/*/src) $(shell find rpc-toolkit/*/src) appmgr/Cargo.toml appmgr/Cargo.lock
 UI_SRC := $(shell find ui/src)
 SETUP_WIZARD_SRC := $(shell find setup-wizard/src)
@@ -14,6 +15,7 @@ clean:
 	rm -f eos.img
 	rm -f ubuntu.img
 	rm -f product_key.txt
+	rm -f system-images/**/*.tar
 	sudo rm -f $(EMBASSY_BINS)
 	rm -rf ui/node_modules
 	rm -rf ui/www
@@ -24,10 +26,17 @@ clean:
 	rm -rf patch-db/client/node_modules
 	rm -rf patch-db/client/dist
 
-eos.img: $(EMBASSY_SRC)
+eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar
 	! test -f eos.img || rm eos.img
 	./build/make-image.sh
 
+system-images/compat/compat.tar: $(COMPAT_SRC)
+	cd system-images/compat && ./build.sh
+	cd system-images/compat && DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/compat --platform=linux/arm64 -o type=docker,dest=compat.tar .
+
+system-images/utils/utils.tar: $(UTILS_SRC)
+	cd system-images/utils && DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/utils --platform=linux/arm64 -o type=docker,dest=utils.tar .
+
 ubuntu.img:
 	wget -O ubuntu.img.xz https://cdimage.ubuntu.com/releases/21.04/release/ubuntu-21.04-preinstalled-server-arm64+raspi.img.xz
 	unxz ubuntu.img.xz
@@ -43,7 +52,7 @@ $(EMBASSY_BINS): $(APPMGR_SRC)
 ui/node_modules: ui/package.json
 	npm --prefix ui install
 
-ui/www: $(UI_SRC) ui/node_modules patch-db/client/dist ui/config.json
+ui/www: $(UI_SRC) ui/node_modules patch-db/client patch-db/client/dist ui/config.json
 	npm --prefix ui run build-prod
 
 ui/config.json:

appmgr/src/bin/embassy-init.rs

@@ -4,16 +4,19 @@ use embassy::context::rpc::RpcContextConfig;
 use embassy::context::{DiagnosticContext, SetupContext};
 use embassy::db::model::ServerStatus;
 use embassy::disk::main::DEFAULT_PASSWORD;
+use embassy::install::PKG_DOCKER_DIR;
 use embassy::middleware::cors::cors;
 use embassy::middleware::diagnostic::diagnostic;
 use embassy::middleware::encrypt::encrypt;
 #[cfg(feature = "avahi")]
 use embassy::net::mdns::MdnsController;
+use embassy::shutdown::Shutdown;
 use embassy::sound::MARIO_COIN;
 use embassy::util::logger::EmbassyLogger;
 use embassy::util::Invoke;
 use embassy::{Error, ResultExt};
 use http::StatusCode;
+use nix::sys::socket::shutdown;
 use rpc_toolkit::rpc_server;
 use tokio::process::Command;
 use tracing::instrument;
@@ -76,7 +79,7 @@ async fn init(cfg_path: Option<&str>) -> Result<(), Error> {
     }
 
     embassy::disk::main::load(
-        tokio::fs::read_to_string("/embassy-os/disk.guid")
+        tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for zfs pool - keeps track of the disk that goes with your embassy
             .await?
             .trim(),
         cfg.zfs_pool_name(),
@@ -124,8 +127,13 @@ async fn init(cfg_path: Option<&str>) -> Result<(), Error> {
         .invoke(embassy::ErrorKind::Docker)
         .await?;
     tracing::info!("Mounted Docker Data");
-    embassy::install::load_images(cfg.datadir()).await?;
+
+    embassy::install::load_images(cfg.datadir().join(PKG_DOCKER_DIR)).await?;
     tracing::info!("Loaded Docker Images");
+    // Loading system images
+    embassy::install::load_images("/var/lib/embassy/system-images").await?;
+    tracing::info!("Loaded System Docker Images");
+
     embassy::ssh::sync_keys_from_db(&secret_store, "/root/.ssh/authorized_keys").await?;
     tracing::info!("Synced SSH Keys");
 
@@ -175,7 +183,7 @@ async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
 }
 
 #[instrument]
-async fn inner_main(cfg_path: Option<&str>) -> Result<(), Error> {
+async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
     embassy::sound::BEP.play().await?;
 
     run_script_if_exists("/embassy-os/preinit.sh").await;
@@ -204,6 +212,7 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<(), Error> {
                 .invoke(embassy::ErrorKind::Nginx)
                 .await?;
             let ctx = DiagnosticContext::init(cfg_path, e).await?;
+            let mut shutdown_recv = ctx.shutdown.subscribe();
             rpc_server!({
                 command: embassy::diagnostic_api,
                 context: ctx.clone(),
@@ -221,11 +230,17 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<(), Error> {
             })
             .await
             .with_kind(embassy::ErrorKind::Network)?;
-            Ok::<_, Error>(())
+
+            Ok::<_, Error>(
+                shutdown_recv
+                    .recv()
+                    .await
+                    .with_kind(embassy::ErrorKind::Network)?,
+            )
         })()
         .await
     } else {
-        Ok(())
+        Ok(None)
     };
 
     run_script_if_exists("/embassy-os/postinit.sh").await;
@@ -255,7 +270,8 @@ fn main() {
     };
 
     match res {
-        Ok(_) => (),
+        Ok(Some(shutdown)) => shutdown.execute(),
+        Ok(None) => (),
         Err(e) => {
             eprintln!("{}", e.source);
             tracing::debug!("{:?}", e.source);

appmgr/src/install/mod.rs

@@ -1,23 +1,11 @@
 use std::collections::{BTreeMap, BTreeSet};
+use std::ffi::OsStr;
 use std::io::SeekFrom;
 use std::path::Path;
 use std::process::Stdio;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
-use color_eyre::eyre::{self, eyre};
-use emver::VersionRange;
-use futures::TryStreamExt;
-use http::StatusCode;
-use patch_db::{DbHandle, LockType};
-use reqwest::Response;
-use rpc_toolkit::command;
-use tokio::fs::{File, OpenOptions};
-use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt};
-use tokio::process::Command;
-use tokio_stream::wrappers::ReadDirStream;
-use tracing::instrument;
-
 use self::cleanup::cleanup_failed;
 use crate::context::RpcContext;
 use crate::db::model::{
@@ -38,6 +26,19 @@ use crate::util::io::copy_and_shutdown;
 use crate::util::{display_none, display_serializable, AsyncFileExt, Version};
 use crate::volume::asset_dir;
 use crate::{Error, ResultExt};
+use color_eyre::eyre::{self, eyre};
+use emver::VersionRange;
+use futures::future::BoxFuture;
+use futures::{FutureExt, StreamExt, TryStreamExt};
+use http::StatusCode;
+use patch_db::{DbHandle, LockType};
+use reqwest::Response;
+use rpc_toolkit::command;
+use tokio::fs::{DirEntry, File, OpenOptions};
+use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt};
+use tokio::process::Command;
+use tokio_stream::wrappers::ReadDirStream;
+use tracing::instrument;
 
 pub mod cleanup;
 pub mod progress;
@@ -766,58 +767,62 @@ async fn handle_recovered_package(
 }
 
 #[instrument(skip(datadir))]
-pub async fn load_images<P: AsRef<Path>>(datadir: P) -> Result<(), Error> {
+pub fn load_images<'a, P: AsRef<Path> + 'a + Send + Sync>(
-    let docker_dir = datadir.as_ref().join(PKG_DOCKER_DIR);
+    datadir: P,
-    if tokio::fs::metadata(&docker_dir).await.is_ok() {
+) -> BoxFuture<'a, Result<(), Error>> {
-        ReadDirStream::new(tokio::fs::read_dir(&docker_dir).await?)
+    async move {
-            .map_err(|e| {
+        let docker_dir = datadir.as_ref();
-                Error::new(
+        if tokio::fs::metadata(&docker_dir).await.is_ok() {
-                    eyre::Report::from(e).wrap_err(format!("{:?}", &docker_dir)),
+            ReadDirStream::new(tokio::fs::read_dir(&docker_dir).await?)
-                    crate::ErrorKind::Filesystem,
+                .map(|r| {
-                )
+                    r.with_ctx(|_| (crate::ErrorKind::Filesystem, format!("{:?}", &docker_dir)))
-            })
+                })
-            .try_for_each_concurrent(None, |pkg_id| async move {
+                .try_for_each(|entry| async move {
-                ReadDirStream::new(tokio::fs::read_dir(pkg_id.path()).await?)
+                    let m = entry.metadata().await?;
-                    .map_err(|e| {
+                    if m.is_file() {
-                        Error::new(
+                        if entry.path().extension().and_then(|ext| ext.to_str()) == Some("tar") {
-                            eyre::Report::from(e).wrap_err(pkg_id.path().display().to_string()),
+                            let mut load = Command::new("docker")
-                            crate::ErrorKind::Filesystem,
+                                .arg("load")
-                        )
+                                .stdin(Stdio::piped())
-                    })
+                                .stderr(Stdio::piped())
-                    .try_for_each_concurrent(None, |version| async move {
+                                .spawn()?;
-                        let mut load = Command::new("docker")
+                            let load_in = load.stdin.take().ok_or_else(|| {
-                            .arg("load")
+                                Error::new(
-                            .stdin(Stdio::piped())
+                                    eyre!("Could not write to stdin of docker load"),
-                            .stderr(Stdio::piped())
+                                    crate::ErrorKind::Docker,
-                            .spawn()?;
+                                )
-                        let load_in = load.stdin.take().ok_or_else(|| {
+                            })?;
-                            Error::new(
+                            let mut docker_rdr = File::open(&entry.path()).await?;
-                                eyre!("Could not write to stdin of docker load"),
+                            copy_and_shutdown(&mut docker_rdr, load_in).await?;
-                                crate::ErrorKind::Docker,
+                            let res = load.wait_with_output().await?;
-                            )
+                            if !res.status.success() {
-                        })?;
+                                Err(Error::new(
-                        let mut docker_rdr = File::open(version.path().join("image.tar")).await?;
+                                    eyre!(
-                        copy_and_shutdown(&mut docker_rdr, load_in).await?;
+                                        "{}",
-                        let res = load.wait_with_output().await?;
+                                        String::from_utf8(res.stderr).unwrap_or_else(|e| format!(
-                        if !res.status.success() {
+                                            "Could not parse stderr: {}",
-                            Err(Error::new(
+                                            e
-                                eyre!(
+                                        ))
-                                    "{}",
+                                    ),
-                                    String::from_utf8(res.stderr).unwrap_or_else(|e| format!(
+                                    crate::ErrorKind::Docker,
-                                        "Could not parse stderr: {}",
+                                ))
-                                        e
+                            } else {
-                                    ))
+                                Ok(())
-                                ),
+                            }
-                                crate::ErrorKind::Docker,
-                            ))
                         } else {
                             Ok(())
                         }
-                    })
+                    } else if m.is_dir() {
-                    .await
+                        load_images(entry.path()).await?;
-            })
+                        Ok(())
-            .await
+                    } else {
-    } else {
+                        Ok(())
-        Ok(())
+                    }
+                })
+                .await
+        } else {
+            Ok(())
+        }
     }
+    .boxed()
 }

build/initialization.sh

@@ -26,13 +26,17 @@ sed -i 's/#allow-interfaces=eth0/allow-interfaces=eth0,wlan0/g' /etc/avahi/avahi
 echo "auto wlan0" > /etc/network/interfaces
 echo "iface wlan0 inet dhcp" >> /etc/network/interfaces
 mkdir -p /etc/nginx/ssl
+
+# fix to suppress docker warning, fixed in 21.xx release of docker cli: https://github.com/docker/cli/pull/2934
+mkdir /root/.docker
+touch /root/.docker/config.json
+
 docker run --privileged --rm tonistiigi/binfmt --install all
 docker network create -d bridge --subnet 172.18.0.1/16 start9 || true
 echo '{ "storage-driver": "zfs" }' > /etc/docker/daemon.json
 mkdir -p /etc/embassy
 hostnamectl set-hostname "embassy"
 systemctl enable embassyd.service embassy-init.service
-echo 'overlayroot="tmpfs"' > /etc/overlayroot.local.conf
 cat << EOF > /etc/tor/torrc
 SocksPort 0.0.0.0:9050
 SocksPolicy accept 127.0.0.1
@@ -42,6 +46,7 @@ ControlPort 9051
 CookieAuthentication 1
 EOF
 
+echo 'overlayroot="tmpfs"' > /etc/overlayroot.local.conf
 systemctl disable initialization.service
 sync
 reboot

build/write-image.sh

@@ -46,6 +46,10 @@ sudo cp *.service /tmp/eos-mnt/etc/systemd/system/
 
 cd ..
 
+# Copy system images
+sudo mkdir -p /tmp/eos-mnt/var/lib/embassy/system-images
+sudo cp system-images/**/*.tar /tmp/eos-mnt/var/lib/embassy/system-images
+
 # after performing npm run build
 sudo mkdir -p /tmp/eos-mnt/var/www/html
 sudo cp -R ui/www /tmp/eos-mnt/var/www/html/main