fix: take into account service dependencies when loading the service containers and load them in correct order

* fix --arch flag to fall back to emulation when native image unavailable, always infer hardware requirement for arch

* better handling of arch filter

* dont cancel in-progress commit workflows and abstract common setup

* cli improvements

fix group handling

* fix cli publish

* alpha.19

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>

.github/actions/setup-build/action.yml

@@ -5,10 +5,6 @@ inputs:
   nodejs-version:
     description: Node.js version
     required: true
-  use-tmpfs:
-    description: Mount tmpfs (for fast runners)
-    required: false
-    default: "false"
   setup-python:
     description: Set up Python
     required: false
@@ -51,14 +47,10 @@ runs:
         sudo rm -rf /usr/share/swift
         sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
-    - name: Mount tmpfs
+    # BuildJet runners lack /opt/hostedtoolcache, which setup-python and setup-qemu expect
-      if: inputs.use-tmpfs == 'true'
+    - name: Ensure hostedtoolcache exists
       shell: bash
-      run: sudo mount -t tmpfs tmpfs .
+      run: sudo mkdir -p /opt/hostedtoolcache && sudo chown $USER:$USER /opt/hostedtoolcache
-
-    - uses: actions/checkout@v4
-      with:
-        submodules: recursive
 
     - name: Set up Python
       if: inputs.setup-python == 'true'
@@ -69,6 +61,8 @@ runs:
     - uses: actions/setup-node@v4
       with:
         node-version: ${{ inputs.nodejs-version }}
+        cache: npm
+        cache-dependency-path: "**/package-lock.json"
 
     - name: Set up Docker QEMU
       if: inputs.setup-docker == 'true'

.github/workflows/start-cli.yaml

@@ -65,10 +65,15 @@ jobs:
           }}
     runs-on: ${{ fromJson('["ubuntu-latest", "buildjet-32vcpu-ubuntu-2204"]')[github.event.inputs.runner == 'fast'] }}
     steps:
+      - name: Mount tmpfs
+        if: ${{ github.event.inputs.runner == 'fast' }}
+        run: sudo mount -t tmpfs tmpfs .
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: ./.github/actions/setup-build
         with:
           nodejs-version: ${{ env.NODEJS_VERSION }}
-          use-tmpfs: ${{ github.event.inputs.runner == 'fast' }}
 
       - name: Make
         run: TARGET=${{ matrix.triple }} make cli

.github/workflows/start-registry.yaml

@@ -61,10 +61,15 @@ jobs:
           }}
     runs-on: ${{ fromJson('["ubuntu-latest", "buildjet-32vcpu-ubuntu-2204"]')[github.event.inputs.runner == 'fast'] }}
     steps:
+      - name: Mount tmpfs
+        if: ${{ github.event.inputs.runner == 'fast' }}
+        run: sudo mount -t tmpfs tmpfs .
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: ./.github/actions/setup-build
         with:
           nodejs-version: ${{ env.NODEJS_VERSION }}
-          use-tmpfs: ${{ github.event.inputs.runner == 'fast' }}
 
       - name: Make
         run: make registry-deb

.github/workflows/start-tunnel.yaml

@@ -61,10 +61,15 @@ jobs:
           }}
     runs-on: ${{ fromJson('["ubuntu-latest", "buildjet-32vcpu-ubuntu-2204"]')[github.event.inputs.runner == 'fast'] }}
     steps:
+      - name: Mount tmpfs
+        if: ${{ github.event.inputs.runner == 'fast' }}
+        run: sudo mount -t tmpfs tmpfs .
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: ./.github/actions/setup-build
         with:
           nodejs-version: ${{ env.NODEJS_VERSION }}
-          use-tmpfs: ${{ github.event.inputs.runner == 'fast' }}
 
       - name: Make
         run: make tunnel-deb

.github/workflows/startos-iso.yaml

@@ -27,7 +27,7 @@ on:
           - x86_64-nonfree
           - aarch64
           - aarch64-nonfree
-          - raspberrypi
+          # - raspberrypi
           - riscv64
       deploy:
         type: choice
@@ -91,10 +91,15 @@ jobs:
         )[github.event.inputs.runner == 'fast']
       }}
     steps:
+      - name: Mount tmpfs
+        if: ${{ github.event.inputs.runner == 'fast' }}
+        run: sudo mount -t tmpfs tmpfs .
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: ./.github/actions/setup-build
         with:
           nodejs-version: ${{ env.NODEJS_VERSION }}
-          use-tmpfs: ${{ github.event.inputs.runner == 'fast' }}
           setup-python: "true"
 
       - name: Make
@@ -113,13 +118,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # TODO: re-add "raspberrypi" to the platform list below
         platform: >-
           ${{
             fromJson(
               format(
                 '[
                   ["{0}"],
-                  ["x86_64", "x86_64-nonfree", "aarch64", "aarch64-nonfree", "riscv64", "raspberrypi"]
+                  ["x86_64", "x86_64-nonfree", "aarch64", "aarch64-nonfree", "riscv64"]
                 ]',
                 github.event.inputs.platform || 'ALL'
               )
@@ -183,6 +189,10 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"    # Pre-cached tool cache (Go, Node, etc.)
         if: ${{ github.event.inputs.runner != 'fast' }}
 
+      # BuildJet runners lack /opt/hostedtoolcache, which setup-qemu expects
+      - name: Ensure hostedtoolcache exists
+        run: sudo mkdir -p /opt/hostedtoolcache && sudo chown $USER:$USER /opt/hostedtoolcache
+
       - name: Set up docker QEMU
         uses: docker/setup-qemu-action@v3
 

.github/workflows/test.yaml

@@ -24,6 +24,9 @@ jobs:
     if: github.event.pull_request.draft != true
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: ./.github/actions/setup-build
         with:
           nodejs-version: ${{ env.NODEJS_VERSION }}

core/src/service/service_map.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
@@ -88,17 +88,77 @@ impl ServiceMap {
 
     #[instrument(skip_all)]
     pub async fn init(&self, ctx: &RpcContext) -> Result<(), Error> {
-        let ids = ctx.db.peek().await.as_public().as_package_data().keys()?;
+        let db = ctx.db.peek().await;
-        let mut jobs = FuturesUnordered::new();
+        let ids = db.as_public().as_package_data().keys()?;
+        
+        // Build dependency map for all packages
+        let mut dependencies: BTreeMap<PackageId, BTreeSet<PackageId>> = BTreeMap::new();
         for id in &ids {
-            jobs.push(self.load(ctx, id, LoadDisposition::Retry));
+            if let Some(pde) = db.as_public().as_package_data().as_idx(&id) {
-        }
+                let deps: BTreeSet<PackageId> = pde
-        while let Some(res) = jobs.next().await {
+                    .as_current_dependencies()
-            if let Err(e) = res {
+                    .de()?
-                tracing::error!("Error loading installed package as service: {e}");
+                    .0
-                tracing::debug!("{e:?}");
+                    .keys()
+                    .cloned()
+                    .collect();
+                dependencies.insert(id.clone(), deps);
             }
         }
+        
+        let mut remaining: BTreeSet<PackageId> = ids.iter().cloned().collect();
+        
+        while !remaining.is_empty() {
+            // Find packages with no remaining dependencies
+            let can_load: Vec<PackageId> = remaining
+                .iter()
+                .filter(|pkg_id| {
+                    // A package can be loaded if none of its dependencies are still in the remaining set
+                    if let Some(deps) = dependencies.get(*pkg_id) {
+                        !deps.iter().any(|dep| remaining.contains(dep))
+                    } else {
+                        true
+                    }
+                })
+                .cloned()
+                .collect();
+            
+            if can_load.is_empty() {
+                // Dependency cycle detected, load remaining packages anyway
+                tracing::warn!(
+                    "Dependency cycle detected, loading remaining packages in arbitrary order"
+                );
+                let mut jobs = FuturesUnordered::new();
+                for id in &remaining {
+                    jobs.push(self.load(ctx, id, LoadDisposition::Retry));
+                }
+                while let Some(res) = jobs.next().await {
+                    if let Err(e) = res {
+                        tracing::error!("Error loading installed package as service: {e}");
+                        tracing::debug!("{e:?}");
+                    }
+                }
+                break;
+            }
+            
+            // Remove from remaining set
+            for id in &can_load {
+                remaining.remove(id);
+            }
+            
+            // Load packages with no remaining dependencies concurrently
+            let mut jobs = FuturesUnordered::new();
+            for id in &can_load {
+                jobs.push(self.load(ctx, id, LoadDisposition::Retry));
+            }
+            while let Some(res) = jobs.next().await {
+                if let Err(e) = res {
+                    tracing::error!("Error loading installed package as service: {e}");
+                    tracing::debug!("{e:?}");
+                }
+            }
+        }
+        
         Ok(())
     }