consolidate and streamline build (#2469)

* consolidate and streamline build

* fix workflow syntax

* fix workflow syntax

* fix workflow syntax

* fix workflow syntax

* fix build scripts

* only build platform-specific system images

* fix build script

* more build fixes

* fix

* fix compat build for x86

* wat

* checkout

* Prevent rebuild of compiled artifacts

* Update startos-iso.yaml

* Update startos-iso.yaml

* fix raspi build

* handle missing platform better

* reduce arm vcpus

* remove arch and platform from fe config, add to patch db

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
This commit is contained in:
Aiden McClelland
2023-10-23 14:40:00 -06:00
committed by GitHub
parent 63ab739b3d
commit 8ea3c3c29e
62 changed files with 1052 additions and 368 deletions

23
backend/Cargo.lock generated
View File

@@ -1896,28 +1896,6 @@ version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
[[package]]
name = "git-version"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6b0decc02f4636b9ccad390dcbe77b722a77efedfa393caf8379a51d5c61899"
dependencies = [
"git-version-macro",
"proc-macro-hack",
]
[[package]]
name = "git-version-macro"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe69f1cbdb6e28af2bac214e943b99ce8a0a06b447d15d3e61161b0423139f3f"
dependencies = [
"proc-macro-hack",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "glob"
version = "0.3.1"
@@ -4938,7 +4916,6 @@ dependencies = [
"emver",
"fd-lock-rs",
"futures",
"git-version",
"gpt",
"helpers",
"hex",

View File

@@ -15,6 +15,7 @@ name = "start-os"
readme = "README.md"
repository = "https://github.com/Start9Labs/start-os"
version = "0.3.5"
license = "MIT"
[lib]
name = "startos"
@@ -31,7 +32,7 @@ cli = []
daemon = []
default = ["cli", "sdk", "daemon", "js_engine"]
dev = []
podman = []
docker = []
sdk = []
unstable = ["console-subscriber", "tokio/tracing"]
@@ -76,7 +77,6 @@ emver = { version = "0.1.7", git = "https://github.com/Start9Labs/emver-rs.git",
] }
fd-lock-rs = "0.1.4"
futures = "0.3.28"
git-version = "0.3.5"
gpt = "3.1.0"
helpers = { path = "../libs/helpers" }
hex = "0.4.3"

View File

@@ -3,11 +3,6 @@
set -e
shopt -s expand_aliases
if [ -z "$OS_ARCH" ]; then
>&2 echo '$OS_ARCH is required'
exit 1
fi
if [ -z "$ARCH" ]; then
ARCH=$(uname -m)
fi
@@ -23,27 +18,17 @@ if tty -s; then
fi
cd ..
FLAGS=""
FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
RUSTFLAGS=""
if [[ "$ENVIRONMENT" =~ (^|-)podman($|-) ]]; then
FLAGS="podman,$FLAGS"
fi
if [[ "$ENVIRONMENT" =~ (^|-)unstable($|-) ]]; then
FLAGS="unstable,$FLAGS"
RUSTFLAGS="$RUSTFLAGS --cfg tokio_unstable"
fi
if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then
FLAGS="dev,$FLAGS"
fi
alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "OS_ARCH=$OS_ARCH" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
set +e
fail=
echo "FLAGS=\"$FLAGS\""
echo "FEATURES=\"$FEATURES\""
echo "RUSTFLAGS=\"$RUSTFLAGS\""
rust-gnu-builder sh -c "(cd backend && cargo build --release --features avahi-alias,$FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
rust-gnu-builder sh -c "(cd backend && cargo build --release --features avahi-alias,$FEATURES --locked --target=$ARCH-unknown-linux-gnu)"
if test $? -ne 0; then
fail=true
fi

View File

@@ -11,8 +11,8 @@ fi
frontend="../frontend/dist/static"
[ -d "$frontend" ] || mkdir -p "$frontend"
if [ -z "$OS_ARCH" ]; then
export OS_ARCH=$(uname -m)
if [ -z "$PLATFORM" ]; then
export PLATFORM=$(uname -m)
fi
cargo install --path=. --no-default-features --features=js_engine,sdk,cli --locked

View File

@@ -1,13 +1,9 @@
use clap::Arg;
use rpc_toolkit::command;
use rpc_toolkit::run_cli;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{command, run_cli, Context};
use serde_json::Value;
use crate::context::CliContext;
use crate::procedure::js_scripts::ExecuteArgs;
use crate::s9pk::manifest::PackageId;
use crate::util::logger::EmbassyLogger;
use crate::util::serde::{display_serializable, parse_stdin_deserializable};
use crate::version::{Current, VersionT};
use crate::Error;
@@ -16,6 +12,9 @@ lazy_static::lazy_static! {
static ref VERSION_STRING: String = Current::new().semver().to_string();
}
struct DenoContext;
impl Context for DenoContext {}
#[command(subcommands(execute, sandbox))]
fn deno_api() -> Result<(), Error> {
Ok(())
@@ -103,16 +102,8 @@ fn inner_main() -> Result<(), Error> {
command: deno_api,
app: app => app
.name("StartOS Deno Executor")
.version(&**VERSION_STRING)
.arg(
clap::Arg::with_name("config")
.short('c')
.long("config")
.takes_value(true),
),
context: matches => {
CliContext::init(matches)?
},
.version(&**VERSION_STRING),
context: _m => DenoContext,
exit: |e: RpcError| {
match e.data {
Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),

View File

@@ -17,7 +17,7 @@ use crate::net::web_server::WebServer;
use crate::shutdown::Shutdown;
use crate::sound::CHIME;
use crate::util::Invoke;
use crate::{Error, ErrorKind, ResultExt, OS_ARCH};
use crate::{Error, ErrorKind, ResultExt, PLATFORM};
#[instrument(skip_all)]
async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
@@ -30,19 +30,19 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
Command::new("ln")
.arg("-sf")
.arg("/usr/lib/embassy/scripts/fake-apt")
.arg("/usr/lib/startos/scripts/fake-apt")
.arg("/usr/local/bin/apt")
.invoke(crate::ErrorKind::OpenSsh)
.await?;
Command::new("ln")
.arg("-sf")
.arg("/usr/lib/embassy/scripts/fake-apt")
.arg("/usr/lib/startos/scripts/fake-apt")
.arg("/usr/local/bin/apt-get")
.invoke(crate::ErrorKind::OpenSsh)
.await?;
Command::new("ln")
.arg("-sf")
.arg("/usr/lib/embassy/scripts/fake-apt")
.arg("/usr/lib/startos/scripts/fake-apt")
.arg("/usr/local/bin/aptitude")
.invoke(crate::ErrorKind::OpenSsh)
.await?;
@@ -177,7 +177,7 @@ async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
#[instrument(skip_all)]
async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
if OS_ARCH == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
Command::new("sync").invoke(ErrorKind::Filesystem).await?;
crate::sound::SHUTDOWN.play().await?;

View File

@@ -1,4 +1,4 @@
use std::borrow::{Borrow, Cow};
use std::borrow::{Cow};
use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::fmt::Debug;
@@ -105,7 +105,7 @@ where
rng: &mut R,
timeout: &Option<Duration>,
) -> Result<Value, Self::Error> {
self.gen_with(self.default_spec().borrow(), rng, timeout)
self.gen_with(self.default_spec(), rng, timeout)
}
}

View File

@@ -4,6 +4,7 @@ use std::sync::Arc;
use chrono::{DateTime, Utc};
use emver::VersionRange;
use imbl_value::InternedString;
use ipnet::{Ipv4Net, Ipv6Net};
use isocountry::CountryCode;
use itertools::Itertools;
@@ -23,6 +24,7 @@ use crate::s9pk::manifest::{Manifest, PackageId};
use crate::status::Status;
use crate::util::Version;
use crate::version::{Current, VersionT};
use crate::{ARCH, PLATFORM};
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
@@ -38,6 +40,8 @@ impl Database {
let lan_address = account.hostname.lan_address().parse().unwrap();
Database {
server_info: ServerInfo {
arch: get_arch(),
platform: get_platform(),
id: account.server_id.clone(),
version: Current::new().semver().into(),
hostname: account.hostname.no_dot_host_name(),
@@ -87,10 +91,22 @@ impl Database {
pub type DatabaseModel = Model<Database>;
fn get_arch() -> InternedString {
(*ARCH).into()
}
fn get_platform() -> InternedString {
(&*PLATFORM).into()
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct ServerInfo {
#[serde(default = "get_arch")]
pub arch: InternedString,
#[serde(default = "get_platform")]
pub platform: InternedString,
pub id: String,
pub hostname: String,
pub version: Version,

View File

@@ -23,7 +23,7 @@ pub async fn update_firmware() -> Result<RequiresReboot, Error> {
if product_name.is_empty() {
return Ok(RequiresReboot(false));
}
let firmware_dir = Path::new("/usr/lib/embassy/firmware").join(&product_name);
let firmware_dir = Path::new("/usr/lib/startos/firmware").join(&product_name);
if tokio::fs::metadata(&firmware_dir).await.is_ok() {
let current_firmware = String::from_utf8(
Command::new("dmidecode")

View File

@@ -263,6 +263,9 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
tracing::info!("Mounted Logs");
let tmp_dir = cfg.datadir().join("package-data/tmp");
if should_rebuild && tokio::fs::metadata(&tmp_dir).await.is_ok() {
tokio::fs::remove_dir_all(&tmp_dir).await?;
}
if tokio::fs::metadata(&tmp_dir).await.is_err() {
tokio::fs::create_dir_all(&tmp_dir).await?;
}
@@ -275,9 +278,6 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
.datadir()
.join(format!("package-data/tmp/{CONTAINER_TOOL}"));
let tmp_docker_exists = tokio::fs::metadata(&tmp_docker).await.is_ok();
if should_rebuild && tmp_docker_exists {
tokio::fs::remove_dir_all(&tmp_docker).await?;
}
if CONTAINER_TOOL == "docker" {
Command::new("systemctl")
.arg("stop")
@@ -309,7 +309,7 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
}
tracing::info!("Loading System Docker Images");
crate::install::load_images("/usr/lib/embassy/system-images").await?;
crate::install::load_images("/usr/lib/startos/system-images").await?;
tracing::info!("Loaded System Docker Images");
tracing::info!("Loading Package Docker Images");

View File

@@ -5,12 +5,18 @@ pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com";
pub const BUFFER_SIZE: usize = 1024;
pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
pub const TARGET: &str = current_platform::CURRENT_PLATFORM;
pub const OS_ARCH: &str = env!("OS_ARCH");
lazy_static::lazy_static! {
pub static ref ARCH: &'static str = {
let (arch, _) = TARGET.split_once("-").unwrap();
arch
};
pub static ref PLATFORM: String = {
if let Ok(platform) = std::fs::read_to_string("/usr/lib/startos/PLATFORM.txt") {
platform
} else {
ARCH.to_string()
}
};
}
pub mod account;

View File

@@ -145,7 +145,7 @@ pub struct JournalctlEntry {
#[serde(rename = "__REALTIME_TIMESTAMP")]
pub timestamp: String,
#[serde(rename = "MESSAGE")]
#[serde(deserialize_with = "deserialize_string_or_utf8_array")]
#[serde(deserialize_with = "deserialize_log_message")]
pub message: String,
#[serde(rename = "__CURSOR")]
pub cursor: String,
@@ -164,7 +164,7 @@ impl JournalctlEntry {
}
}
fn deserialize_string_or_utf8_array<'de, D: serde::de::Deserializer<'de>>(
fn deserialize_log_message<'de, D: serde::de::Deserializer<'de>>(
deserializer: D,
) -> std::result::Result<String, D::Error> {
struct Visitor;
@@ -177,13 +177,7 @@ fn deserialize_string_or_utf8_array<'de, D: serde::de::Deserializer<'de>>(
where
E: serde::de::Error,
{
Ok(v.to_owned())
}
fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(v)
Ok(v.trim().to_owned())
}
fn visit_unit<E>(self) -> Result<Self::Value, E>
where
@@ -201,6 +195,7 @@ fn deserialize_string_or_utf8_array<'de, D: serde::de::Deserializer<'de>>(
.flatten()
.collect::<Result<Vec<u8>, _>>()?,
)
.map(|s| s.trim().to_owned())
.map_err(serde::de::Error::custom)
}
}
@@ -374,12 +369,12 @@ pub async fn journalctl(
cmd.arg(format!("_COMM={}", SYSTEM_UNIT));
}
LogSource::Container(id) => {
#[cfg(feature = "podman")]
#[cfg(not(feature = "docker"))]
cmd.arg(format!(
"SYSLOG_IDENTIFIER={}",
DockerProcedure::container_name(&id, None)
));
#[cfg(not(feature = "podman"))]
#[cfg(feature = "docker")]
cmd.arg(format!(
"CONTAINER_NAME={}",
DockerProcedure::container_name(&id, None)

View File

@@ -813,7 +813,7 @@ impl LongRunning {
socket_path: &Path,
) -> Result<tokio::process::Command, Error> {
const INIT_EXEC: &str = "/start9/bin/embassy_container_init";
const BIND_LOCATION: &str = "/usr/lib/embassy/container/";
const BIND_LOCATION: &str = "/usr/lib/startos/container/";
tracing::trace!("setup_long_running_docker_cmd");
remove_container(container_name, true).await?;

View File

@@ -23,7 +23,7 @@ pub fn with_query_params(ctx: RpcContext, mut url: Url) -> Url {
"os.compat",
&crate::version::Current::new().compat().to_string(),
)
.append_pair("os.arch", crate::OS_ARCH)
.append_pair("os.arch", &*crate::PLATFORM)
.append_pair("hardware.arch", &*crate::ARCH)
.append_pair("hardware.ram", &ctx.hardware.ram.to_string());

View File

@@ -9,7 +9,7 @@ use crate::init::{STANDBY_MODE_PATH, SYSTEM_REBUILD_PATH};
use crate::sound::SHUTDOWN;
use crate::util::docker::CONTAINER_TOOL;
use crate::util::{display_none, Invoke};
use crate::{Error, OS_ARCH};
use crate::{Error, PLATFORM};
#[derive(Debug, Clone)]
pub struct Shutdown {
@@ -60,7 +60,7 @@ impl Shutdown {
tracing::debug!("{:?}", e);
}
}
if OS_ARCH != "raspberrypi" || self.restart {
if &*PLATFORM != "raspberrypi" || self.restart {
if let Err(e) = SHUTDOWN.play().await {
tracing::error!("Error Playing Shutdown Song: {}", e);
tracing::debug!("{:?}", e);
@@ -68,7 +68,7 @@ impl Shutdown {
}
});
drop(rt);
if OS_ARCH == "raspberrypi" {
if &*PLATFORM == "raspberrypi" {
if !self.restart {
std::fs::write(STANDBY_MODE_PATH, "").unwrap();
Command::new("sync").spawn().unwrap().wait().unwrap();

View File

@@ -25,7 +25,7 @@ use crate::sound::{
};
use crate::update::latest_information::LatestInformation;
use crate::util::Invoke;
use crate::{Error, ErrorKind, ResultExt, OS_ARCH};
use crate::{Error, ErrorKind, ResultExt, PLATFORM};
mod latest_information;
@@ -231,7 +231,7 @@ impl EosUrl {
.host_str()
.ok_or_else(|| Error::new(eyre!("Could not get host of base"), ErrorKind::ParseUrl))?;
let version: &Version = &self.version;
Ok(format!("{host}::{version}/{OS_ARCH}/")
Ok(format!("{host}::{version}/{}/", &*PLATFORM)
.parse()
.map_err(|_| Error::new(eyre!("Could not parse path"), ErrorKind::ParseUrl))?)
}
@@ -297,7 +297,7 @@ async fn sync_boot() -> Result<(), Error> {
.await?
.wait()
.await?;
if OS_ARCH != "raspberrypi" {
if &*PLATFORM != "raspberrypi" {
let dev_mnt =
MountGuard::mount(&Bind::new("/dev"), "/media/embassy/next/dev", ReadWrite).await?;
let sys_mnt =

View File

@@ -7,14 +7,14 @@ use tokio::process::Command;
use crate::util::Invoke;
#[cfg(not(feature = "podman"))]
#[cfg(feature = "docker")]
pub const CONTAINER_TOOL: &str = "docker";
#[cfg(feature = "podman")]
#[cfg(not(feature = "docker"))]
pub const CONTAINER_TOOL: &str = "podman";
#[cfg(not(feature = "podman"))]
#[cfg(feature = "docker")]
pub const CONTAINER_DATADIR: &str = "/var/lib/docker";
#[cfg(feature = "podman")]
#[cfg(not(feature = "docker"))]
pub const CONTAINER_DATADIR: &str = "/var/lib/containers";
pub struct DockerImageSha(String);

View File

@@ -182,8 +182,7 @@ pub async fn init(db: &PatchDb, secrets: &PgPool) -> Result<(), Error> {
Ok(())
}
pub const COMMIT_HASH: &str =
git_version::git_version!(args = ["--always", "--abbrev=40", "--dirty=-modified"]);
pub const COMMIT_HASH: &str = include_str!("../../../GIT_HASH.txt");
#[command(rename = "git-info", local, metadata(authenticated = false))]
pub fn git_info() -> Result<&'static str, Error> {

View File

@@ -857,7 +857,7 @@ export const action = {
},
/**
* Created this test because of issue
* https://github.com/Start9Labs/embassy-os/issues/1737
* https://github.com/Start9Labs/start-os/issues/1737
* which that we couldn't create a dir that was deeply nested, and the parents where
* not created yet. Found this out during the migrations, where the parent would die.
* @param {*} effects
@@ -933,7 +933,7 @@ export const action = {
},
/**
* Created this test because of issue
* https://github.com/Start9Labs/embassy-os/issues/2121
* https://github.com/Start9Labs/start-os/issues/2121
* That the empty in the create dies
* @param {*} effects
* @param {*} _input

View File

@@ -15,7 +15,7 @@ docker run -d --rm --name=tmp_postgres -e POSTGRES_PASSWORD=password -v $TMP_DIR
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' tmp_postgres)
DATABASE_URL=postgres://postgres:password@$PG_IP/postgres cargo sqlx migrate run
DATABASE_URL=postgres://postgres:password@$PG_IP/postgres OS_ARCH=$(uname -m) cargo sqlx prepare -- --lib --profile=test
DATABASE_URL=postgres://postgres:password@$PG_IP/postgres PLATFORM=$(uname -m) cargo sqlx prepare -- --lib --profile=test
)
docker stop tmp_postgres