mirror of
https://github.com/Start9Labs/start-os.git
synced 2026-03-31 04:23:40 +00:00
Refactor/project structure (#3085)
* refactor project structure * environment-based default registry * fix tests * update build container * use docker platform for iso build emulation * simplify compat * Fix docker platform spec in run-compat.sh * handle riscv compat * fix bug with dep error exists attr * undo removal of sorting * use qemu for iso stage --------- Co-authored-by: Mariusz Kogen <k0gen@pm.me> Co-authored-by: Matt Hill <mattnine@protonmail.com>
This commit is contained in:
Aiden McClelland
864
core/src/s9pk/v2/pack.rs
Normal file
864
core/src/s9pk/v2/pack.rs
Normal file
@@ -0,0 +1,864 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::sync::{Arc, LazyLock, OnceLock};
|
||||
|
||||
use clap::Parser;
|
||||
use futures::future::{BoxFuture, ready};
|
||||
use futures::{FutureExt, TryStreamExt};
|
||||
use imbl_value::InternedString;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::OnceCell;
|
||||
use tokio_stream::wrappers::ReadDirStream;
|
||||
use tracing::{debug, warn};
|
||||
use ts_rs::TS;
|
||||
|
||||
use crate::context::CliContext;
|
||||
use crate::dependencies::{DependencyMetadata, MetadataSrc};
|
||||
use crate::prelude::*;
|
||||
use crate::rpc_continuations::Guid;
|
||||
use crate::s9pk::S9pk;
|
||||
use crate::s9pk::git_hash::GitHash;
|
||||
use crate::s9pk::manifest::Manifest;
|
||||
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
|
||||
use crate::s9pk::merkle_archive::source::http::HttpSource;
|
||||
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
|
||||
use crate::s9pk::merkle_archive::source::{
|
||||
ArchiveSource, DynFileSource, DynRead, FileSource, TmpSource, into_dyn_read,
|
||||
};
|
||||
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
|
||||
use crate::s9pk::v2::SIG_CONTEXT;
|
||||
use crate::util::io::{TmpDir, create_file, open_file};
|
||||
use crate::util::serde::IoFormat;
|
||||
use crate::util::{DataUrl, Invoke, PathOrUrl, VersionString, new_guid};
|
||||
use crate::{ImageId, PackageId};
|
||||
|
||||
pub static PREFER_DOCKER: OnceLock<bool> = OnceLock::new();
|
||||
|
||||
pub static CONTAINER_TOOL: LazyLock<&'static str> = LazyLock::new(|| {
|
||||
if *PREFER_DOCKER.get_or_init(|| false) {
|
||||
if std::process::Command::new("which")
|
||||
.arg("docker")
|
||||
.stdout(Stdio::null())
|
||||
.status()
|
||||
.map_or(false, |o| o.success())
|
||||
{
|
||||
"docker"
|
||||
} else {
|
||||
"podman"
|
||||
}
|
||||
} else {
|
||||
"podman"
|
||||
}
|
||||
});
|
||||
pub static CONTAINER_DATADIR: LazyLock<&'static str> = LazyLock::new(|| {
|
||||
if *CONTAINER_TOOL == "docker" {
|
||||
"/var/lib/docker"
|
||||
} else {
|
||||
"/var/lib/containers"
|
||||
}
|
||||
});
|
||||
|
||||
pub struct SqfsDir {
|
||||
path: PathBuf,
|
||||
tmpdir: Arc<TmpDir>,
|
||||
sqfs: OnceCell<MultiCursorFile>,
|
||||
}
|
||||
impl SqfsDir {
|
||||
pub fn new(path: PathBuf, tmpdir: Arc<TmpDir>) -> Self {
|
||||
Self {
|
||||
path,
|
||||
tmpdir,
|
||||
sqfs: OnceCell::new(),
|
||||
}
|
||||
}
|
||||
async fn file(&self) -> Result<&MultiCursorFile, Error> {
|
||||
self.sqfs
|
||||
.get_or_try_init(|| async move {
|
||||
let guid = Guid::new();
|
||||
let path = self.tmpdir.join(guid.as_ref()).with_extension("squashfs");
|
||||
if self.path.extension().and_then(|s| s.to_str()) == Some("tar") {
|
||||
tar2sqfs(&self.path)?
|
||||
.input(Some(&mut open_file(&self.path).await?))
|
||||
.invoke(ErrorKind::Filesystem)
|
||||
.await?;
|
||||
} else {
|
||||
Command::new("mksquashfs")
|
||||
.arg(&self.path)
|
||||
.arg(&path)
|
||||
.arg("-quiet")
|
||||
.invoke(ErrorKind::Filesystem)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(MultiCursorFile::from(
|
||||
open_file(&path)
|
||||
.await
|
||||
.with_ctx(|_| (ErrorKind::Filesystem, path.display()))?,
|
||||
))
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum PackSource {
|
||||
Buffered(Arc<[u8]>),
|
||||
File(PathBuf),
|
||||
Squashfs(Arc<SqfsDir>),
|
||||
}
|
||||
impl FileSource for PackSource {
|
||||
type Reader = DynRead;
|
||||
type SliceReader = DynRead;
|
||||
async fn size(&self) -> Result<u64, Error> {
|
||||
match self {
|
||||
Self::Buffered(a) => Ok(a.len() as u64),
|
||||
Self::File(f) => Ok(tokio::fs::metadata(f)
|
||||
.await
|
||||
.with_ctx(|_| (ErrorKind::Filesystem, f.display()))?
|
||||
.len()),
|
||||
Self::Squashfs(dir) => dir
|
||||
.file()
|
||||
.await
|
||||
.with_ctx(|_| (ErrorKind::Filesystem, dir.path.display()))?
|
||||
.size()
|
||||
.await
|
||||
.or_not_found("file metadata"),
|
||||
}
|
||||
}
|
||||
async fn reader(&self) -> Result<Self::Reader, Error> {
|
||||
match self {
|
||||
Self::Buffered(a) => Ok(into_dyn_read(FileSource::reader(a).await?)),
|
||||
Self::File(f) => Ok(into_dyn_read(FileSource::reader(f).await?)),
|
||||
Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read),
|
||||
}
|
||||
}
|
||||
async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
|
||||
match self {
|
||||
Self::Buffered(a) => Ok(into_dyn_read(FileSource::slice(a, position, size).await?)),
|
||||
Self::File(f) => Ok(into_dyn_read(FileSource::slice(f, position, size).await?)),
|
||||
Self::Squashfs(dir) => dir
|
||||
.file()
|
||||
.await?
|
||||
.fetch(position, size)
|
||||
.await
|
||||
.map(into_dyn_read),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl From<PackSource> for DynFileSource {
|
||||
fn from(value: PackSource) -> Self {
|
||||
DynFileSource::new(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Parser)]
|
||||
pub struct PackParams {
|
||||
pub path: Option<PathBuf>,
|
||||
#[arg(short, long)]
|
||||
pub output: Option<PathBuf>,
|
||||
#[arg(long)]
|
||||
pub javascript: Option<PathBuf>,
|
||||
#[arg(long)]
|
||||
pub icon: Option<PathBuf>,
|
||||
#[arg(long)]
|
||||
pub license: Option<PathBuf>,
|
||||
#[arg(long, conflicts_with = "no-assets")]
|
||||
pub assets: Option<PathBuf>,
|
||||
#[arg(long, conflicts_with = "assets")]
|
||||
pub no_assets: bool,
|
||||
#[arg(long, help = "Architecture Mask")]
|
||||
pub arch: Vec<InternedString>,
|
||||
}
|
||||
impl PackParams {
|
||||
fn path(&self) -> &Path {
|
||||
self.path.as_deref().unwrap_or(Path::new("."))
|
||||
}
|
||||
fn output(&self, id: &PackageId) -> PathBuf {
|
||||
self.output
|
||||
.as_ref()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.path().join(id).with_extension("s9pk"))
|
||||
}
|
||||
fn javascript(&self) -> PathBuf {
|
||||
self.javascript
|
||||
.as_ref()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.path().join("javascript"))
|
||||
}
|
||||
async fn icon(&self) -> Result<PathBuf, Error> {
|
||||
if let Some(icon) = &self.icon {
|
||||
Ok(icon.clone())
|
||||
} else {
|
||||
ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
|
||||
.try_filter(|x| {
|
||||
ready(
|
||||
x.path()
|
||||
.file_stem()
|
||||
.map_or(false, |s| s.eq_ignore_ascii_case("icon")),
|
||||
)
|
||||
})
|
||||
.map_err(Error::from)
|
||||
.try_fold(
|
||||
Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)),
|
||||
|acc, x| async move {
|
||||
match acc {
|
||||
Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
|
||||
Err(e) => Ok({
|
||||
let path = x.path();
|
||||
if path
|
||||
.file_stem()
|
||||
.map_or(false, |s| s.eq_ignore_ascii_case("icon"))
|
||||
{
|
||||
Ok(path)
|
||||
} else {
|
||||
Err(e)
|
||||
}
|
||||
}),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
async fn license(&self) -> Result<PathBuf, Error> {
|
||||
if let Some(license) = &self.license {
|
||||
Ok(license.clone())
|
||||
} else {
|
||||
ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
|
||||
.try_filter(|x| {
|
||||
ready(
|
||||
x.path()
|
||||
.file_stem()
|
||||
.map_or(false, |s| s.eq_ignore_ascii_case("license")),
|
||||
)
|
||||
})
|
||||
.map_err(Error::from)
|
||||
.try_fold(
|
||||
Err(Error::new(eyre!("license not found"), ErrorKind::NotFound)),
|
||||
|acc, x| async move {
|
||||
match acc {
|
||||
Ok(_) => Err(Error::new(eyre!("multiple licenses found in working directory, please specify which to use with `--license`"), ErrorKind::InvalidRequest)),
|
||||
Err(e) => Ok({
|
||||
let path = x.path();
|
||||
if path
|
||||
.file_stem()
|
||||
.map_or(false, |s| s.eq_ignore_ascii_case("license"))
|
||||
{
|
||||
Ok(path)
|
||||
} else {
|
||||
Err(e)
|
||||
}
|
||||
}),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
fn assets(&self) -> PathBuf {
|
||||
self.assets
|
||||
.as_ref()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.path().join("assets"))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export)]
|
||||
pub struct ImageConfig {
|
||||
pub source: ImageSource,
|
||||
#[ts(type = "string[]")]
|
||||
pub arch: BTreeSet<InternedString>,
|
||||
#[ts(type = "string | null")]
|
||||
pub emulate_missing_as: Option<InternedString>,
|
||||
}
|
||||
impl Default for ImageConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
source: ImageSource::Packed,
|
||||
arch: BTreeSet::new(),
|
||||
emulate_missing_as: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
struct CliImageConfig {
|
||||
#[arg(long, conflicts_with("docker-tag"))]
|
||||
docker_build: bool,
|
||||
#[arg(long, requires("docker-build"))]
|
||||
dockerfile: Option<PathBuf>,
|
||||
#[arg(long, requires("docker-build"))]
|
||||
workdir: Option<PathBuf>,
|
||||
#[arg(long, conflicts_with_all(["dockerfile", "workdir"]))]
|
||||
docker_tag: Option<String>,
|
||||
#[arg(long)]
|
||||
arch: Vec<InternedString>,
|
||||
#[arg(long)]
|
||||
emulate_missing_as: Option<InternedString>,
|
||||
}
|
||||
impl TryFrom<CliImageConfig> for ImageConfig {
|
||||
type Error = clap::Error;
|
||||
fn try_from(value: CliImageConfig) -> Result<Self, Self::Error> {
|
||||
let res = Self {
|
||||
source: if value.docker_build {
|
||||
ImageSource::DockerBuild {
|
||||
dockerfile: value.dockerfile,
|
||||
workdir: value.workdir,
|
||||
build_args: None,
|
||||
}
|
||||
} else if let Some(tag) = value.docker_tag {
|
||||
ImageSource::DockerTag(tag)
|
||||
} else {
|
||||
ImageSource::Packed
|
||||
},
|
||||
arch: value.arch.into_iter().collect(),
|
||||
emulate_missing_as: value.emulate_missing_as,
|
||||
};
|
||||
res.emulate_missing_as
|
||||
.as_ref()
|
||||
.map(|a| {
|
||||
if !res.arch.contains(a) {
|
||||
Err(clap::Error::raw(
|
||||
clap::error::ErrorKind::InvalidValue,
|
||||
"`emulate-missing-as` must match one of the provided `arch`es",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.transpose()?;
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
impl clap::Args for ImageConfig {
|
||||
fn augment_args(cmd: clap::Command) -> clap::Command {
|
||||
CliImageConfig::augment_args(cmd)
|
||||
}
|
||||
fn augment_args_for_update(cmd: clap::Command) -> clap::Command {
|
||||
CliImageConfig::augment_args_for_update(cmd)
|
||||
}
|
||||
}
|
||||
impl clap::FromArgMatches for ImageConfig {
|
||||
fn from_arg_matches(matches: &clap::ArgMatches) -> Result<Self, clap::Error> {
|
||||
Self::try_from(CliImageConfig::from_arg_matches(matches)?)
|
||||
}
|
||||
fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
|
||||
*self = Self::try_from(CliImageConfig::from_arg_matches(matches)?)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(untagged)]
|
||||
#[ts(export)]
|
||||
pub enum BuildArg {
|
||||
String(String),
|
||||
EnvVar { env: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export)]
|
||||
pub enum ImageSource {
|
||||
Packed,
|
||||
#[serde(rename_all = "camelCase")]
|
||||
DockerBuild {
|
||||
#[ts(optional)]
|
||||
workdir: Option<PathBuf>,
|
||||
#[ts(optional)]
|
||||
dockerfile: Option<PathBuf>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional)]
|
||||
build_args: Option<BTreeMap<String, BuildArg>>,
|
||||
},
|
||||
DockerTag(String),
|
||||
// Recipe(DirRecipe),
|
||||
}
|
||||
impl ImageSource {
|
||||
pub fn ingredients(&self) -> Vec<PathBuf> {
|
||||
match self {
|
||||
Self::Packed => Vec::new(),
|
||||
Self::DockerBuild {
|
||||
dockerfile,
|
||||
workdir,
|
||||
..
|
||||
} => {
|
||||
vec![
|
||||
workdir
|
||||
.as_deref()
|
||||
.unwrap_or(Path::new("."))
|
||||
.join(dockerfile.as_deref().unwrap_or(Path::new("Dockerfile"))),
|
||||
]
|
||||
}
|
||||
Self::DockerTag(_) => Vec::new(),
|
||||
}
|
||||
}
|
||||
#[instrument(skip_all)]
|
||||
pub fn load<'a, S: From<TmpSource<PackSource>> + FileSource + Clone>(
|
||||
&'a self,
|
||||
tmp_dir: Arc<TmpDir>,
|
||||
id: &'a PackageId,
|
||||
version: &'a VersionString,
|
||||
image_id: &'a ImageId,
|
||||
arch: &'a str,
|
||||
into: &'a mut DirectoryContents<S>,
|
||||
) -> BoxFuture<'a, Result<(), Error>> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "PascalCase")]
|
||||
struct DockerImageConfig {
|
||||
env: Vec<String>,
|
||||
#[serde(default)]
|
||||
working_dir: PathBuf,
|
||||
#[serde(default)]
|
||||
user: String,
|
||||
entrypoint: Option<Vec<String>>,
|
||||
cmd: Option<Vec<String>>,
|
||||
}
|
||||
async move {
|
||||
match self {
|
||||
ImageSource::Packed => Ok(()),
|
||||
ImageSource::DockerBuild {
|
||||
workdir,
|
||||
dockerfile,
|
||||
build_args,
|
||||
} => {
|
||||
let workdir = workdir.as_deref().unwrap_or(Path::new("."));
|
||||
let dockerfile = dockerfile
|
||||
.clone()
|
||||
.unwrap_or_else(|| workdir.join("Dockerfile"));
|
||||
let docker_platform = if arch == "x86_64" {
|
||||
"--platform=linux/amd64".to_owned()
|
||||
} else if arch == "aarch64" {
|
||||
"--platform=linux/arm64".to_owned()
|
||||
} else {
|
||||
format!("--platform=linux/{arch}")
|
||||
};
|
||||
// docker buildx build ${path} -o type=image,name=start9/${id}
|
||||
let tag = format!("start9/{id}/{image_id}:{}", new_guid());
|
||||
let mut command = Command::new(*CONTAINER_TOOL);
|
||||
if *CONTAINER_TOOL == "docker" {
|
||||
command.arg("buildx");
|
||||
}
|
||||
command
|
||||
.arg("build")
|
||||
.arg(workdir)
|
||||
.arg("-f")
|
||||
.arg(dockerfile)
|
||||
.arg("-t")
|
||||
.arg(&tag)
|
||||
.arg(&docker_platform)
|
||||
.arg("--build-arg")
|
||||
.arg(format!("ARCH={}", arch));
|
||||
|
||||
// add build arguments
|
||||
if let Some(build_args) = build_args {
|
||||
for (key, value) in build_args {
|
||||
let build_arg_value = match value {
|
||||
BuildArg::String(val) => val.to_string(),
|
||||
BuildArg::EnvVar { env } => {
|
||||
match std::env::var(&env) {
|
||||
Ok(val) => val,
|
||||
Err(_) => continue, // skip if env var not set or invalid
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
command
|
||||
.arg("--build-arg")
|
||||
.arg(format!("{}={}", key, build_arg_value));
|
||||
}
|
||||
}
|
||||
|
||||
command
|
||||
.arg("-o")
|
||||
.arg("type=docker,dest=-")
|
||||
.capture(false)
|
||||
.pipe(Command::new(*CONTAINER_TOOL).arg("load"))
|
||||
.invoke(ErrorKind::Docker)
|
||||
.await?;
|
||||
ImageSource::DockerTag(tag.clone())
|
||||
.load(tmp_dir, id, version, image_id, arch, into)
|
||||
.await?;
|
||||
Command::new(*CONTAINER_TOOL)
|
||||
.arg("rmi")
|
||||
.arg("-f")
|
||||
.arg(&tag)
|
||||
.invoke(ErrorKind::Docker)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
ImageSource::DockerTag(tag) => {
|
||||
let docker_platform = if arch == "x86_64" {
|
||||
"--platform=linux/amd64".to_owned()
|
||||
} else if arch == "aarch64" {
|
||||
"--platform=linux/arm64".to_owned()
|
||||
} else {
|
||||
format!("--platform=linux/{arch}")
|
||||
};
|
||||
let container = String::from_utf8(
|
||||
Command::new(*CONTAINER_TOOL)
|
||||
.arg("create")
|
||||
.arg(&docker_platform)
|
||||
.arg(&tag)
|
||||
.invoke(ErrorKind::Docker)
|
||||
.await?,
|
||||
)?;
|
||||
let container = container.trim();
|
||||
let config = serde_json::from_slice::<DockerImageConfig>(
|
||||
&Command::new(*CONTAINER_TOOL)
|
||||
.arg("container")
|
||||
.arg("inspect")
|
||||
.arg("--format")
|
||||
.arg("{{json .Config}}")
|
||||
.arg(container)
|
||||
.invoke(ErrorKind::Docker)
|
||||
.await?,
|
||||
)
|
||||
.with_kind(ErrorKind::Deserialization)?;
|
||||
let base_path = Path::new("images").join(arch).join(image_id);
|
||||
into.insert_path(
|
||||
base_path.with_extension("json"),
|
||||
Entry::file(
|
||||
TmpSource::new(
|
||||
tmp_dir.clone(),
|
||||
PackSource::Buffered(
|
||||
serde_json::to_vec(&ImageMetadata {
|
||||
workdir: if config.working_dir == Path::new("") {
|
||||
"/".into()
|
||||
} else {
|
||||
config.working_dir
|
||||
},
|
||||
user: if config.user.is_empty() {
|
||||
"root".into()
|
||||
} else {
|
||||
config.user.into()
|
||||
},
|
||||
entrypoint: config.entrypoint,
|
||||
cmd: config.cmd,
|
||||
})
|
||||
.with_kind(ErrorKind::Serialization)?
|
||||
.into(),
|
||||
),
|
||||
)
|
||||
.into(),
|
||||
),
|
||||
)?;
|
||||
into.insert_path(
|
||||
base_path.with_extension("env"),
|
||||
Entry::file(
|
||||
TmpSource::new(
|
||||
tmp_dir.clone(),
|
||||
PackSource::Buffered(config.env.join("\n").into_bytes().into()),
|
||||
)
|
||||
.into(),
|
||||
),
|
||||
)?;
|
||||
let dest = tmp_dir
|
||||
.join(Guid::new().as_ref())
|
||||
.with_extension("squashfs");
|
||||
|
||||
Command::new(*CONTAINER_TOOL)
|
||||
.arg("export")
|
||||
.arg(container)
|
||||
.pipe(&mut tar2sqfs(&dest)?)
|
||||
.capture(false)
|
||||
.invoke(ErrorKind::Docker)
|
||||
.await?;
|
||||
Command::new(*CONTAINER_TOOL)
|
||||
.arg("rm")
|
||||
.arg(container)
|
||||
.invoke(ErrorKind::Docker)
|
||||
.await?;
|
||||
into.insert_path(
|
||||
base_path.with_extension("squashfs"),
|
||||
Entry::file(TmpSource::new(tmp_dir.clone(), PackSource::File(dest)).into()),
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
fn tar2sqfs(dest: impl AsRef<Path>) -> Result<Command, Error> {
|
||||
let dest = dest.as_ref();
|
||||
|
||||
Ok({
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let mut command = Command::new("tar2sqfs");
|
||||
command.arg("-q").arg(&dest);
|
||||
command
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let directory = dest
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new("/"))
|
||||
.to_path_buf();
|
||||
let mut command = Command::new(*CONTAINER_TOOL);
|
||||
command
|
||||
.arg("run")
|
||||
.arg("-i")
|
||||
.arg("--rm")
|
||||
.arg("--mount")
|
||||
.arg(format!("type=bind,src={},dst=/data", directory.display()))
|
||||
.arg("ghcr.io/start9labs/sdk/utils:latest")
|
||||
.arg("tar2sqfs")
|
||||
.arg("-q")
|
||||
.arg(Path::new("/data").join(&dest.file_name().unwrap_or_default()));
|
||||
command
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export)]
|
||||
pub struct ImageMetadata {
|
||||
pub workdir: PathBuf,
|
||||
#[ts(type = "string")]
|
||||
pub user: InternedString,
|
||||
pub entrypoint: Option<Vec<String>>,
|
||||
pub cmd: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
|
||||
let tmp_dir = Arc::new(TmpDir::new().await?);
|
||||
let mut files = DirectoryContents::<TmpSource<PackSource>>::new();
|
||||
let js_dir = params.javascript();
|
||||
let manifest: Arc<[u8]> = Command::new("node")
|
||||
.arg("-e")
|
||||
.arg(format!(
|
||||
"console.log(JSON.stringify(require('{}/index.js').manifest))",
|
||||
js_dir.display()
|
||||
))
|
||||
.invoke(ErrorKind::Javascript)
|
||||
.await?
|
||||
.into();
|
||||
files.insert(
|
||||
"manifest.json".into(),
|
||||
Entry::file(TmpSource::new(
|
||||
tmp_dir.clone(),
|
||||
PackSource::Buffered(manifest.clone()),
|
||||
)),
|
||||
);
|
||||
let icon = params.icon().await?;
|
||||
let icon_ext = icon
|
||||
.extension()
|
||||
.or_not_found("icon file extension")?
|
||||
.to_string_lossy();
|
||||
files.insert(
|
||||
InternedString::from_display(&lazy_format!("icon.{}", icon_ext)),
|
||||
Entry::file(TmpSource::new(tmp_dir.clone(), PackSource::File(icon))),
|
||||
);
|
||||
files.insert(
|
||||
"LICENSE.md".into(),
|
||||
Entry::file(TmpSource::new(
|
||||
tmp_dir.clone(),
|
||||
PackSource::File(params.license().await?),
|
||||
)),
|
||||
);
|
||||
files.insert(
|
||||
"javascript.squashfs".into(),
|
||||
Entry::file(TmpSource::new(
|
||||
tmp_dir.clone(),
|
||||
PackSource::Squashfs(Arc::new(SqfsDir::new(js_dir, tmp_dir.clone()))),
|
||||
)),
|
||||
);
|
||||
|
||||
let mut s9pk = S9pk::new(
|
||||
MerkleArchive::new(files, ctx.developer_key()?.clone(), SIG_CONTEXT),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let manifest = s9pk.as_manifest_mut();
|
||||
manifest.git_hash = Some(GitHash::from_path(params.path()).await?);
|
||||
if !params.arch.is_empty() {
|
||||
let arches = match manifest.hardware_requirements.arch.take() {
|
||||
Some(a) => params
|
||||
.arch
|
||||
.iter()
|
||||
.filter(|x| a.contains(*x))
|
||||
.cloned()
|
||||
.collect(),
|
||||
None => params.arch.iter().cloned().collect(),
|
||||
};
|
||||
manifest
|
||||
.images
|
||||
.values_mut()
|
||||
.for_each(|c| c.arch = c.arch.intersection(&arches).cloned().collect());
|
||||
manifest.hardware_requirements.arch = Some(arches);
|
||||
}
|
||||
|
||||
if !params.no_assets {
|
||||
let assets_dir = params.assets();
|
||||
s9pk.as_archive_mut().contents_mut().insert_path(
|
||||
"assets.squashfs",
|
||||
Entry::file(TmpSource::new(
|
||||
tmp_dir.clone(),
|
||||
PackSource::Squashfs(Arc::new(SqfsDir::new(assets_dir, tmp_dir.clone()))),
|
||||
)),
|
||||
)?;
|
||||
}
|
||||
|
||||
s9pk.load_images(tmp_dir.clone()).await?;
|
||||
|
||||
let mut to_insert = Vec::new();
|
||||
for (id, dependency) in &mut s9pk.as_manifest_mut().dependencies.0 {
|
||||
if let Some((title, icon)) = match dependency.metadata.take() {
|
||||
Some(MetadataSrc::Metadata(metadata)) => {
|
||||
let icon = match metadata.icon {
|
||||
PathOrUrl::Path(path) => DataUrl::from_path(path).await?,
|
||||
PathOrUrl::Url(url) => {
|
||||
if url.scheme() == "http" || url.scheme() == "https" {
|
||||
DataUrl::from_response(ctx.client.get(url).send().await?).await?
|
||||
} else if url.scheme() == "data" {
|
||||
url.as_str().parse()?
|
||||
} else {
|
||||
return Err(Error::new(
|
||||
eyre!("unknown scheme: {}", url.scheme()),
|
||||
ErrorKind::InvalidRequest,
|
||||
));
|
||||
}
|
||||
}
|
||||
};
|
||||
Some((metadata.title, icon))
|
||||
}
|
||||
Some(MetadataSrc::S9pk(Some(s9pk))) => {
|
||||
let s9pk = match s9pk {
|
||||
PathOrUrl::Path(path) => {
|
||||
S9pk::deserialize(&MultiCursorFile::from(open_file(path).await?), None)
|
||||
.await?
|
||||
.into_dyn()
|
||||
}
|
||||
PathOrUrl::Url(url) => {
|
||||
if url.scheme() == "http" || url.scheme() == "https" {
|
||||
S9pk::deserialize(
|
||||
&Arc::new(HttpSource::new(ctx.client.clone(), url).await?),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.into_dyn()
|
||||
} else {
|
||||
return Err(Error::new(
|
||||
eyre!("unknown scheme: {}", url.scheme()),
|
||||
ErrorKind::InvalidRequest,
|
||||
));
|
||||
}
|
||||
}
|
||||
};
|
||||
Some((
|
||||
s9pk.as_manifest().title.clone(),
|
||||
s9pk.icon_data_url().await?,
|
||||
))
|
||||
}
|
||||
Some(MetadataSrc::S9pk(None)) | None => {
|
||||
warn!("no metadata specified for {id}, leaving metadata empty");
|
||||
None
|
||||
}
|
||||
} {
|
||||
let dep_path = Path::new("dependencies").join(id);
|
||||
to_insert.push((
|
||||
dep_path.join("metadata.json"),
|
||||
Entry::file(TmpSource::new(
|
||||
tmp_dir.clone(),
|
||||
PackSource::Buffered(
|
||||
IoFormat::Json.to_vec(&DependencyMetadata { title })?.into(),
|
||||
),
|
||||
)),
|
||||
));
|
||||
to_insert.push((
|
||||
dep_path
|
||||
.join("icon")
|
||||
.with_extension(icon.canonical_ext().unwrap_or("ico")),
|
||||
Entry::file(TmpSource::new(
|
||||
tmp_dir.clone(),
|
||||
PackSource::Buffered(icon.data.into_owned().into()),
|
||||
)),
|
||||
));
|
||||
}
|
||||
}
|
||||
for (path, source) in to_insert {
|
||||
s9pk.as_archive_mut()
|
||||
.contents_mut()
|
||||
.insert_path(path, source)?;
|
||||
}
|
||||
|
||||
s9pk.validate_and_filter(None)?;
|
||||
|
||||
s9pk.serialize(
|
||||
&mut create_file(params.output(&s9pk.as_manifest().id)).await?,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
drop(s9pk);
|
||||
|
||||
tmp_dir.gc().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
pub async fn list_ingredients(_: CliContext, params: PackParams) -> Result<Vec<PathBuf>, Error> {
|
||||
let js_path = params.javascript().join("index.js");
|
||||
let manifest: Manifest = match async {
|
||||
serde_json::from_slice(
|
||||
&Command::new("node")
|
||||
.arg("-e")
|
||||
.arg(format!(
|
||||
"console.log(JSON.stringify(require('{}').manifest))",
|
||||
js_path.display()
|
||||
))
|
||||
.invoke(ErrorKind::Javascript)
|
||||
.await?,
|
||||
)
|
||||
.with_kind(ErrorKind::Deserialization)
|
||||
}
|
||||
.await
|
||||
{
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
warn!("failed to load manifest: {e}");
|
||||
debug!("{e:?}");
|
||||
return Ok(vec![js_path, params.icon().await?, params.license().await?]);
|
||||
}
|
||||
};
|
||||
let mut ingredients = vec![js_path, params.icon().await?, params.license().await?];
|
||||
|
||||
for (_, dependency) in manifest.dependencies.0 {
|
||||
match dependency.metadata {
|
||||
Some(MetadataSrc::Metadata(crate::dependencies::Metadata {
|
||||
icon: PathOrUrl::Path(icon),
|
||||
..
|
||||
})) => {
|
||||
ingredients.push(icon);
|
||||
}
|
||||
Some(MetadataSrc::S9pk(Some(PathOrUrl::Path(s9pk)))) => {
|
||||
ingredients.push(s9pk);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
if !params.no_assets {
|
||||
let assets_dir = params.assets();
|
||||
ingredients.push(assets_dir);
|
||||
}
|
||||
|
||||
for image in manifest.images.values() {
|
||||
ingredients.extend(image.source.ingredients());
|
||||
}
|
||||
|
||||
Ok(ingredients)
|
||||
}
|
||||
Reference in New Issue
Block a user