create version graph to handle migrations (#2708)

* create version graph to handle migrations

* Fix some version alpha test

* connect dataVersion api

* rename init fns

* improve types and add tests

* set data version after backup restore

* chore: Add some types tests for version info

* wip: More changes to versionInfo tests

* wip: fix my stupid

* update mocks

* update runtime

* chore: Fix the loop

---------

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>
Co-authored-by: J H <dragondef@gmail.com>
This commit is contained in:
Aiden McClelland
2024-08-15 20:58:53 +00:00
committed by GitHub
parent c704626a39
commit c174b65465
34 changed files with 974 additions and 257 deletions

View File

@@ -284,6 +284,16 @@ function makeEffects(context: EffectContext): Effects {
set: async (options: any) =>
rpcRound("setStore", options) as ReturnType<T.Effects["store"]["set"]>,
} as T.Effects["store"],
getDataVersion() {
return rpcRound("getDataVersion", {}) as ReturnType<
T.Effects["getDataVersion"]
>
},
setDataVersion(...[options]: Parameters<T.Effects["setDataVersion"]>) {
return rpcRound("setDataVersion", options) as ReturnType<
T.Effects["setDataVersion"]
>
},
}
return self
}

View File

@@ -145,9 +145,7 @@ export class SystemForStartOs implements System {
): Promise<unknown> {
switch (options.procedure) {
case "/init": {
const previousVersion =
string.optional().unsafeCast(options.input) || null
return this.abi.init({ effects, previousVersion })
return this.abi.init({ effects })
}
case "/uninit": {
const nextVersion = string.optional().unsafeCast(options.input) || null

View File

@@ -3,7 +3,9 @@ use std::collections::{BTreeMap, BTreeSet};
use chrono::{DateTime, Utc};
use exver::VersionRange;
use imbl_value::InternedString;
use models::{ActionId, DataUrl, HealthCheckId, HostId, PackageId, ServiceInterfaceId};
use models::{
ActionId, DataUrl, HealthCheckId, HostId, PackageId, ServiceInterfaceId, VersionString,
};
use patch_db::json_ptr::JsonPointer;
use patch_db::HasModel;
use reqwest::Url;
@@ -335,6 +337,7 @@ pub struct ActionMetadata {
#[ts(export)]
pub struct PackageDataEntry {
pub state_info: PackageState,
pub data_version: Option<VersionString>,
pub status: Status,
#[ts(type = "string | null")]
pub registry: Option<Url>,

View File

@@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::sync::Arc;
use exver::ExtendedVersion;
use exver::{ExtendedVersion, VersionRange};
use models::ImageId;
use tokio::io::{AsyncRead, AsyncSeek, AsyncWriteExt};
use tokio::process::Command;
@@ -203,6 +203,8 @@ impl From<ManifestV1> for Manifest {
version: ExtendedVersion::from(value.version).into(),
satisfies: BTreeSet::new(),
release_notes: value.release_notes,
can_migrate_from: VersionRange::any(),
can_migrate_to: VersionRange::none(),
license: value.license.into(),
wrapper_repo: value.wrapper_repo,
upstream_repo: value.upstream_repo,

View File

@@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use color_eyre::eyre::eyre;
use exver::Version;
use exver::{Version, VersionRange};
use helpers::const_true;
use imbl_value::InternedString;
pub use models::PackageId;
@@ -37,6 +37,10 @@ pub struct Manifest {
pub satisfies: BTreeSet<VersionString>,
pub release_notes: String,
#[ts(type = "string")]
pub can_migrate_to: VersionRange,
#[ts(type = "string")]
pub can_migrate_from: VersionRange,
#[ts(type = "string")]
pub license: InternedString, // type of license
#[ts(type = "string")]
pub wrapper_repo: Url,
@@ -159,8 +163,8 @@ impl Manifest {
#[ts(export)]
pub struct HardwareRequirements {
#[serde(default)]
#[ts(type = "{ device?: string, processor?: string }")]
pub device: BTreeMap<String, Regex>,
#[ts(type = "{ display?: string, processor?: string }")]
pub device: BTreeMap<String, Regex>, // TODO: array
#[ts(type = "number | null")]
pub ram: Option<u64>,
#[ts(type = "string[] | null")]

View File

@@ -164,6 +164,25 @@ pub fn handler<C: Context>() -> ParentHandler<C> {
// store
.subcommand("getStore", from_fn_async(store::get_store).no_cli())
.subcommand("setStore", from_fn_async(store::set_store).no_cli())
.subcommand(
"setDataVersion",
from_fn_async(store::set_data_version)
.no_display()
.with_call_remote::<ContainerCliContext>(),
)
.subcommand(
"getDataVersion",
from_fn_async(store::get_data_version)
.with_custom_display_fn(|_, v| {
if let Some(v) = v {
println!("{v}")
} else {
println!("N/A")
}
Ok(())
})
.with_call_remote::<ContainerCliContext>(),
)
// system
.subcommand(
"getSystemSmtp",

View File

@@ -1,6 +1,6 @@
use imbl::vector;
use imbl_value::json;
use models::PackageId;
use models::{PackageId, VersionString};
use patch_db::json_ptr::JsonPointer;
use crate::service::effects::callbacks::CallbackHandler;
@@ -91,3 +91,50 @@ pub async fn set_store(
Ok(())
}
#[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct SetDataVersionParams {
#[ts(type = "string")]
version: VersionString,
}
pub async fn set_data_version(
context: EffectContext,
SetDataVersionParams { version }: SetDataVersionParams,
) -> Result<(), Error> {
let context = context.deref()?;
let package_id = &context.seed.id;
context
.seed
.ctx
.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(package_id)
.or_not_found(package_id)?
.as_data_version_mut()
.ser(&Some(version))
})
.await?;
Ok(())
}
pub async fn get_data_version(context: EffectContext) -> Result<Option<VersionString>, Error> {
let context = context.deref()?;
let package_id = &context.seed.id;
context
.seed
.ctx
.db
.peek()
.await
.as_public()
.as_package_data()
.as_idx(package_id)
.or_not_found(package_id)?
.as_data_version()
.de()
}

View File

@@ -173,6 +173,7 @@ impl ServiceMap {
} else {
PackageState::Installing(installing)
},
data_version: None,
status: Status {
configured: false,
main: MainStatus::Stopped,

View File

@@ -30,7 +30,7 @@ import { healthCheck, HealthCheckParams } from "./health/HealthCheck"
import { checkPortListening } from "./health/checkFns/checkPortListening"
import { checkWebUrl, runHealthScript } from "./health/checkFns"
import { List } from "./config/builder/list"
import { Migration } from "./inits/migrations/Migration"
import { VersionInfo, VersionOptions } from "./versionInfo/VersionInfo"
import { Install, InstallFn } from "./inits/setupInstall"
import { setupActions } from "./actions/setupActions"
import { setupDependencyConfig } from "./dependencies/setupDependencyConfig"
@@ -38,9 +38,9 @@ import { SetupBackupsParams, setupBackups } from "./backup/setupBackups"
import { setupInit } from "./inits/setupInit"
import {
EnsureUniqueId,
Migrations,
setupMigrations,
} from "./inits/migrations/setupMigrations"
VersionGraph,
setupVersionGraph,
} from "./versionInfo/setupVersionGraph"
import { Uninstall, UninstallFn, setupUninstall } from "./inits/setupUninstall"
import { setupMain } from "./mainFn"
import { defaultTrigger } from "./trigger/defaultTrigger"
@@ -319,7 +319,7 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
setupActions: (...createdActions: CreatedAction<any, any, any>[]) =>
setupActions<Manifest, Store>(...createdActions),
setupBackups: (...args: SetupBackupsParams<Manifest>) =>
setupBackups<Manifest>(...args),
setupBackups<Manifest>(this.manifest, ...args),
setupConfig: <
ConfigType extends Config<any, Store> | Config<any, never>,
Type extends Record<string, any> = ExtractConfigType<ConfigType>,
@@ -388,7 +388,7 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
}
},
setupInit: (
migrations: Migrations<Manifest, Store>,
versions: VersionGraph<Manifest["version"]>,
install: Install<Manifest, Store>,
uninstall: Uninstall<Manifest, Store>,
setInterfaces: SetInterfaces<Manifest, Store, any, any>,
@@ -399,7 +399,7 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
exposedStore: ExposedStorePaths,
) =>
setupInit<Manifest, Store>(
migrations,
versions,
install,
uninstall,
setInterfaces,
@@ -420,15 +420,13 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
started(onTerm: () => PromiseLike<void>): PromiseLike<void>
}) => Promise<Daemons<Manifest, any>>,
) => setupMain<Manifest, Store>(fn),
setupMigrations: <
Migrations extends Array<Migration<Manifest, Store, any>>,
setupVersionGraph: <
CurrentVersion extends string,
OtherVersions extends Array<VersionInfo<any>>,
>(
...migrations: EnsureUniqueId<Migrations>
) =>
setupMigrations<Manifest, Store, Migrations>(
this.manifest,
...migrations,
),
current: VersionInfo<CurrentVersion>,
...other: EnsureUniqueId<OtherVersions, OtherVersions, CurrentVersion>
) => setupVersionGraph<CurrentVersion, OtherVersions>(current, ...other),
setupProperties:
(
fn: (options: { effects: Effects }) => Promise<T.SdkPropertiesReturn>,
@@ -549,12 +547,9 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
>,
) => List.dynamicText<Store>(getA),
},
Migration: {
of: <Version extends string>(options: {
version: Version & ValidateExVer<Version>
up: (opts: { effects: Effects }) => Promise<void>
down: (opts: { effects: Effects }) => Promise<void>
}) => Migration.of<Manifest, Store, Version>(options),
VersionInfo: {
of: <Version extends string>(options: VersionOptions<Version>) =>
VersionInfo.of<Version>(options),
},
StorePath: pathBuilder<Store>(),
Value: {
@@ -755,15 +750,9 @@ export async function runCommand<Manifest extends T.Manifest>(
},
): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
const commands = splitCommand(command)
const overlay = await Overlay.of(effects, image)
try {
for (let mount of options.mounts || []) {
await overlay.mount(mount.options, mount.path)
}
return await overlay.exec(commands)
} finally {
await overlay.destroy()
}
return Overlay.with(effects, image, options.mounts || [], (overlay) =>
overlay.exec(commands),
)
}
function nullifyProperties(value: T.SdkPropertiesReturn): T.PropertiesReturn {
return Object.fromEntries(

View File

@@ -8,6 +8,7 @@ export type SetupBackupsParams<M extends T.Manifest> = Array<
>
export function setupBackups<M extends T.Manifest>(
manifest: M,
...args: _<SetupBackupsParams<M>>
) {
const backups = Array<Backups<M>>()
@@ -36,6 +37,7 @@ export function setupBackups<M extends T.Manifest>(
for (const backup of backups) {
await backup.build(options.pathMaker).restoreBackup(options)
}
await options.effects.setDataVersion({ version: manifest.version })
}) as T.ExpectedExports.restoreBackup
},
}

View File

@@ -3,7 +3,7 @@ import * as P from "./exver"
// prettier-ignore
export type ValidateVersion<T extends String> =
T extends `-${infer A}` ? never :
T extends `${infer A}-${infer B}` ? ValidateVersion<A> & ValidateVersion<B> :
T extends `${infer A}-${string}` ? ValidateVersion<A> :
T extends `${bigint}` ? unknown :
T extends `${bigint}.${infer A}` ? ValidateVersion<A> :
never
@@ -16,9 +16,9 @@ export type ValidateExVer<T extends string> =
// prettier-ignore
export type ValidateExVers<T> =
T extends [] ? unknown :
T extends [] ? unknown[] :
T extends [infer A, ...infer B] ? ValidateExVer<A & string> & ValidateExVers<B> :
never
never[]
type Anchor = {
type: "Anchor"
@@ -426,6 +426,7 @@ function tests() {
testTypeVersion("12.34.56")
testTypeVersion("1.2-3")
testTypeVersion("1-3")
testTypeVersion("1-alpha")
// @ts-expect-error
testTypeVersion("-3")
// @ts-expect-error

View File

@@ -1,35 +0,0 @@
import { ValidateExVer } from "../../exver"
import * as T from "../../types"
export class Migration<
Manifest extends T.Manifest,
Store,
Version extends string,
> {
constructor(
readonly options: {
version: Version & ValidateExVer<Version>
up: (opts: { effects: T.Effects }) => Promise<void>
down: (opts: { effects: T.Effects }) => Promise<void>
},
) {}
static of<
Manifest extends T.Manifest,
Store,
Version extends string,
>(options: {
version: Version & ValidateExVer<Version>
up: (opts: { effects: T.Effects }) => Promise<void>
down: (opts: { effects: T.Effects }) => Promise<void>
}) {
return new Migration<Manifest, Store, Version>(options)
}
async up(opts: { effects: T.Effects }) {
this.up(opts)
}
async down(opts: { effects: T.Effects }) {
this.down(opts)
}
}

View File

@@ -1,77 +0,0 @@
import { ExtendedVersion } from "../../exver"
import * as T from "../../types"
import { once } from "../../util/once"
import { Migration } from "./Migration"
export class Migrations<Manifest extends T.Manifest, Store> {
private constructor(
readonly manifest: T.Manifest,
readonly migrations: Array<Migration<Manifest, Store, any>>,
) {}
private sortedMigrations = once(() => {
const migrationsAsVersions = (
this.migrations as Array<Migration<Manifest, Store, any>>
)
.map((x) => [ExtendedVersion.parse(x.options.version), x] as const)
.filter(([v, _]) => v.flavor === this.currentVersion().flavor)
migrationsAsVersions.sort((a, b) => a[0].compareForSort(b[0]))
return migrationsAsVersions
})
private currentVersion = once(() =>
ExtendedVersion.parse(this.manifest.version),
)
static of<
Manifest extends T.Manifest,
Store,
Migrations extends Array<Migration<Manifest, Store, any>>,
>(manifest: T.Manifest, ...migrations: EnsureUniqueId<Migrations>) {
return new Migrations(
manifest,
migrations as Array<Migration<Manifest, Store, any>>,
)
}
async init({
effects,
previousVersion,
}: Parameters<T.ExpectedExports.init>[0]) {
if (!!previousVersion) {
const previousVersionExVer = ExtendedVersion.parse(previousVersion)
for (const [_, migration] of this.sortedMigrations()
.filter((x) => x[0].greaterThan(previousVersionExVer))
.filter((x) => x[0].lessThanOrEqual(this.currentVersion()))) {
await migration.up({ effects })
}
}
}
async uninit({
effects,
nextVersion,
}: Parameters<T.ExpectedExports.uninit>[0]) {
if (!!nextVersion) {
const nextVersionExVer = ExtendedVersion.parse(nextVersion)
const reversed = [...this.sortedMigrations()].reverse()
for (const [_, migration] of reversed
.filter((x) => x[0].greaterThan(nextVersionExVer))
.filter((x) => x[0].lessThanOrEqual(this.currentVersion()))) {
await migration.down({ effects })
}
}
}
}
export function setupMigrations<
Manifest extends T.Manifest,
Store,
Migrations extends Array<Migration<Manifest, Store, any>>,
>(manifest: T.Manifest, ...migrations: EnsureUniqueId<Migrations>) {
return Migrations.of<Manifest, Store, Migrations>(manifest, ...migrations)
}
// prettier-ignore
export type EnsureUniqueId<A, B = A, ids = never> =
B extends [] ? A :
B extends [Migration<any, any, infer id>, ...infer Rest] ? (
id extends ids ? "One of the ids are not unique"[] :
EnsureUniqueId<A, Rest, id | ids>
) : "There exists a migration that is not a Migration"[]

View File

@@ -1,14 +1,15 @@
import { DependenciesReceipt } from "../config/setupConfig"
import { ExtendedVersion, VersionRange } from "../exver"
import { SetInterfaces } from "../interfaces/setupInterfaces"
import { ExposedStorePaths } from "../store/setupExposeStore"
import * as T from "../types"
import { Migrations } from "./migrations/setupMigrations"
import { VersionGraph } from "../versionInfo/setupVersionGraph"
import { Install } from "./setupInstall"
import { Uninstall } from "./setupUninstall"
export function setupInit<Manifest extends T.Manifest, Store>(
migrations: Migrations<Manifest, Store>,
versions: VersionGraph<Manifest["version"]>,
install: Install<Manifest, Store>,
uninstall: Uninstall<Manifest, Store>,
setInterfaces: SetInterfaces<Manifest, Store, any, any>,
@@ -23,8 +24,19 @@ export function setupInit<Manifest extends T.Manifest, Store>(
} {
return {
init: async (opts) => {
await migrations.init(opts)
await install.init(opts)
const prev = await opts.effects.getDataVersion()
if (prev) {
await versions.migrate({
effects: opts.effects,
from: ExtendedVersion.parse(prev),
to: versions.currentVersion(),
})
} else {
await install.install(opts)
await opts.effects.setDataVersion({
version: versions.current.options.version,
})
}
await setInterfaces({
...opts,
input: null,
@@ -33,8 +45,18 @@ export function setupInit<Manifest extends T.Manifest, Store>(
await setDependencies({ effects: opts.effects, input: null })
},
uninit: async (opts) => {
await migrations.uninit(opts)
await uninstall.uninit(opts)
if (opts.nextVersion) {
const prev = await opts.effects.getDataVersion()
if (prev) {
await versions.migrate({
effects: opts.effects,
from: ExtendedVersion.parse(prev),
to: ExtendedVersion.parse(opts.nextVersion),
})
}
} else {
await uninstall.uninstall(opts)
}
},
}
}

View File

@@ -11,14 +11,10 @@ export class Install<Manifest extends T.Manifest, Store> {
return new Install(fn)
}
async init({
effects,
previousVersion,
}: Parameters<T.ExpectedExports.init>[0]) {
if (!previousVersion)
await this.fn({
effects,
})
async install({ effects }: Parameters<T.ExpectedExports.init>[0]) {
await this.fn({
effects,
})
}
}

View File

@@ -11,7 +11,7 @@ export class Uninstall<Manifest extends T.Manifest, Store> {
return new Uninstall(fn)
}
async uninit({
async uninstall({
effects,
nextVersion,
}: Parameters<T.ExpectedExports.uninit>[0]) {

View File

@@ -7,22 +7,11 @@ import {
ImageSource,
} from "../types"
export type SDKManifest<
Version extends string,
Satisfies extends string[] = [],
> = {
export type SDKManifest = {
/** The package identifier used by the OS. This must be unique amongst all other known packages */
readonly id: string
/** A human readable service title */
readonly title: string
/** Service version - accepts up to four digits, where the last confirms to revisions necessary for StartOs
* - see documentation: https://github.com/Start9Labs/emver-rs. This value will change with each release of
* the service
*/
readonly version: Version & ValidateExVer<Version>
readonly satisfies?: Satisfies & ValidateExVers<Satisfies>
/** Release notes for the update - can be a string, paragraph or URL */
readonly releaseNotes: string
/** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/
readonly license: string // name of license
/** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this),

View File

@@ -2,6 +2,7 @@ import * as T from "../types"
import { ImageConfig, ImageId, VolumeId } from "../osBindings"
import { SDKManifest, SDKImageConfig } from "./ManifestTypes"
import { SDKVersion } from "../StartSdk"
import { VersionGraph } from "../versionInfo/setupVersionGraph"
/**
* This is an example of a function that takes a manifest and returns a new manifest with additional properties
@@ -21,10 +22,12 @@ export function setupManifest<
assets: AssetTypes[]
images: Record<ImagesTypes, SDKImageConfig>
volumes: VolumesTypes[]
version: Version
},
Satisfies extends string[] = [],
>(manifest: SDKManifest<Version, Satisfies> & Manifest): Manifest & T.Manifest {
>(
manifest: SDKManifest & Manifest,
versions: VersionGraph<Version>,
): Manifest & T.Manifest {
const images = Object.entries(manifest.images).reduce(
(images, [k, v]) => {
v.arch = v.arch || ["aarch64", "x86_64"]
@@ -39,7 +42,11 @@ export function setupManifest<
...manifest,
gitHash: null,
osVersion: SDKVersion,
satisfies: manifest.satisfies || [],
version: versions.current.options.version,
releaseNotes: versions.current.options.releaseNotes,
satisfies: versions.current.options.satisfies || [],
canMigrateTo: versions.canMigrateTo().toString(),
canMigrateFrom: versions.canMigrateFrom().toString(),
images,
alerts: {
install: manifest.alerts?.install || null,

View File

@@ -1,7 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type HardwareRequirements = {
device: { device?: string; processor?: string }
device: { display?: string; processor?: string }
ram: number | null
arch: string[] | null
}

View File

@@ -15,6 +15,8 @@ export type Manifest = {
version: Version
satisfies: Array<Version>
releaseNotes: string
canMigrateTo: string
canMigrateFrom: string
license: string
wrapperRepo: string
upstreamRepo: string

View File

@@ -8,9 +8,11 @@ import type { PackageState } from "./PackageState"
import type { ServiceInterface } from "./ServiceInterface"
import type { ServiceInterfaceId } from "./ServiceInterfaceId"
import type { Status } from "./Status"
import type { Version } from "./Version"
export type PackageDataEntry = {
stateInfo: PackageState
dataVersion: Version | null
status: Status
registry: string | null
developerKey: string

View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type SetDataVersionParams = { version: string }

View File

@@ -132,6 +132,7 @@ export { SessionList } from "./SessionList"
export { Sessions } from "./Sessions"
export { Session } from "./Session"
export { SetConfigured } from "./SetConfigured"
export { SetDataVersionParams } from "./SetDataVersionParams"
export { SetDependenciesParams } from "./SetDependenciesParams"
export { SetHealth } from "./SetHealth"
export { SetMainStatusStatus } from "./SetMainStatusStatus"

View File

@@ -6,6 +6,8 @@ import { Variants } from "../config/builder/variants"
import { ValueSpec } from "../config/configTypes"
import { setupManifest } from "../manifest/setupManifest"
import { StartSdk } from "../StartSdk"
import { VersionGraph } from "../versionInfo/setupVersionGraph"
import { VersionInfo } from "../versionInfo/VersionInfo"
describe("builder tests", () => {
test("text", async () => {
@@ -366,42 +368,48 @@ describe("values", () => {
test("datetime", async () => {
const sdk = StartSdk.of()
.withManifest(
setupManifest({
id: "testOutput",
title: "",
version: "1.0.0:0",
releaseNotes: "",
license: "",
replaces: [],
wrapperRepo: "",
upstreamRepo: "",
supportSite: "",
marketingSite: "",
donationUrl: null,
description: {
short: "",
long: "",
},
containers: {},
images: {},
volumes: [],
assets: [],
alerts: {
install: null,
update: null,
uninstall: null,
restore: null,
start: null,
stop: null,
},
dependencies: {
"remote-test": {
description: "",
optional: true,
s9pk: "https://example.com/remote-test.s9pk",
setupManifest(
{
id: "testOutput",
title: "",
license: "",
wrapperRepo: "",
upstreamRepo: "",
supportSite: "",
marketingSite: "",
donationUrl: null,
description: {
short: "",
long: "",
},
containers: {},
images: {},
volumes: [],
assets: [],
alerts: {
install: null,
update: null,
uninstall: null,
restore: null,
start: null,
stop: null,
},
dependencies: {
"remote-test": {
description: "",
optional: true,
s9pk: "https://example.com/remote-test.s9pk",
},
},
},
}),
VersionGraph.of(
VersionInfo.of({
version: "1.0.0:0",
releaseNotes: "",
migrations: {},
}),
),
),
)
.withStore<{ test: "a" }>()
.build(true)

148
sdk/lib/test/graph.test.ts Normal file
View File

@@ -0,0 +1,148 @@
import { Graph } from "../util/graph"
describe("graph", () => {
{
{
test("findVertex", () => {
const graph = new Graph<string, string>()
const foo = graph.addVertex("foo", [], [])
const bar = graph.addVertex(
"bar",
[{ from: foo, metadata: "foo-bar" }],
[],
)
const baz = graph.addVertex(
"baz",
[{ from: bar, metadata: "bar-baz" }],
[],
)
const qux = graph.addVertex(
"qux",
[{ from: baz, metadata: "baz-qux" }],
[],
)
const match = Array.from(graph.findVertex((v) => v.metadata === "qux"))
expect(match).toHaveLength(1)
expect(match[0]).toBe(qux)
})
test("shortestPathA", () => {
const graph = new Graph<string, string>()
const foo = graph.addVertex("foo", [], [])
const bar = graph.addVertex(
"bar",
[{ from: foo, metadata: "foo-bar" }],
[],
)
const baz = graph.addVertex(
"baz",
[{ from: bar, metadata: "bar-baz" }],
[],
)
const qux = graph.addVertex(
"qux",
[{ from: baz, metadata: "baz-qux" }],
[],
)
graph.addEdge("foo-qux", foo, qux)
expect(graph.shortestPath(foo, qux) || []).toHaveLength(1)
})
test("shortestPathB", () => {
const graph = new Graph<string, string>()
const foo = graph.addVertex("foo", [], [])
const bar = graph.addVertex(
"bar",
[{ from: foo, metadata: "foo-bar" }],
[],
)
const baz = graph.addVertex(
"baz",
[{ from: bar, metadata: "bar-baz" }],
[],
)
const qux = graph.addVertex(
"qux",
[{ from: baz, metadata: "baz-qux" }],
[],
)
graph.addEdge("bar-qux", bar, qux)
expect(graph.shortestPath(foo, qux) || []).toHaveLength(2)
})
test("shortestPathC", () => {
const graph = new Graph<string, string>()
const foo = graph.addVertex("foo", [], [])
const bar = graph.addVertex(
"bar",
[{ from: foo, metadata: "foo-bar" }],
[],
)
const baz = graph.addVertex(
"baz",
[{ from: bar, metadata: "bar-baz" }],
[],
)
const qux = graph.addVertex(
"qux",
[{ from: baz, metadata: "baz-qux" }],
[{ to: foo, metadata: "qux-foo" }],
)
expect(graph.shortestPath(foo, qux) || []).toHaveLength(3)
})
test("bfs", () => {
const graph = new Graph<string, string>()
const foo = graph.addVertex("foo", [], [])
const bar = graph.addVertex(
"bar",
[{ from: foo, metadata: "foo-bar" }],
[],
)
const baz = graph.addVertex(
"baz",
[{ from: bar, metadata: "bar-baz" }],
[],
)
const qux = graph.addVertex(
"qux",
[
{ from: foo, metadata: "foo-qux" },
{ from: baz, metadata: "baz-qux" },
],
[],
)
const bfs = Array.from(graph.breadthFirstSearch(foo))
expect(bfs).toHaveLength(4)
expect(bfs[0]).toBe(foo)
expect(bfs[1]).toBe(bar)
expect(bfs[2]).toBe(qux)
expect(bfs[3]).toBe(baz)
})
test("reverseBfs", () => {
const graph = new Graph<string, string>()
const foo = graph.addVertex("foo", [], [])
const bar = graph.addVertex(
"bar",
[{ from: foo, metadata: "foo-bar" }],
[],
)
const baz = graph.addVertex(
"baz",
[{ from: bar, metadata: "bar-baz" }],
[],
)
const qux = graph.addVertex(
"qux",
[
{ from: foo, metadata: "foo-qux" },
{ from: baz, metadata: "baz-qux" },
],
[],
)
const bfs = Array.from(graph.reverseBreadthFirstSearch(qux))
expect(bfs).toHaveLength(4)
expect(bfs[0]).toBe(qux)
expect(bfs[1]).toBe(foo)
expect(bfs[2]).toBe(baz)
expect(bfs[3]).toBe(bar)
})
}
}
})

View File

@@ -1,45 +1,56 @@
import { StartSdk } from "../StartSdk"
import { setupManifest } from "../manifest/setupManifest"
import { VersionInfo } from "../versionInfo/VersionInfo"
import { VersionGraph } from "../versionInfo/setupVersionGraph"
export type Manifest = any
export const sdk = StartSdk.of()
.withManifest(
setupManifest({
id: "testOutput",
title: "",
version: "1.0:0",
releaseNotes: "",
license: "",
replaces: [],
wrapperRepo: "",
upstreamRepo: "",
supportSite: "",
marketingSite: "",
donationUrl: null,
description: {
short: "",
long: "",
},
containers: {},
images: {},
volumes: [],
assets: [],
alerts: {
install: null,
update: null,
uninstall: null,
restore: null,
start: null,
stop: null,
},
dependencies: {
"remote-test": {
description: "",
optional: false,
s9pk: "https://example.com/remote-test.s9pk",
setupManifest(
{
id: "testOutput",
title: "",
license: "",
replaces: [],
wrapperRepo: "",
upstreamRepo: "",
supportSite: "",
marketingSite: "",
donationUrl: null,
description: {
short: "",
long: "",
},
containers: {},
images: {},
volumes: [],
assets: [],
alerts: {
install: null,
update: null,
uninstall: null,
restore: null,
start: null,
stop: null,
},
dependencies: {
"remote-test": {
description: "",
optional: false,
s9pk: "https://example.com/remote-test.s9pk",
},
},
},
}),
VersionGraph.of(
VersionInfo.of({
version: "1.0.0:0",
releaseNotes: "",
migrations: {},
})
.satisfies("#other:1.0.0:0")
.satisfies("#other:2.0.0:0"),
),
),
)
.withStore<{ storeRoot: { storeLeaf: "value" } }>()
.build(true)

View File

@@ -3,6 +3,7 @@ import {
CheckDependenciesParam,
ExecuteAction,
GetConfiguredParams,
SetDataVersionParams,
SetMainStatus,
} from ".././osBindings"
import { CreateOverlayedImageParams } from ".././osBindings"
@@ -46,6 +47,8 @@ describe("startosTypeValidation ", () => {
restart: undefined,
shutdown: undefined,
setConfigured: {} as SetConfigured,
setDataVersion: {} as SetDataVersionParams,
getDataVersion: undefined,
setHealth: {} as SetHealth,
exposeForDependents: {} as ExposeForDependentsParams,
getSslCertificate: {} as WithCallback<GetSslCertificateParams>,

View File

@@ -102,10 +102,7 @@ export namespace ExpectedExports {
* Every time a package completes an install, this function is called before the main.
* Can be used to do migration like things.
*/
export type init = (options: {
effects: Effects
previousVersion: null | string
}) => Promise<unknown>
export type init = (options: { effects: Effects }) => Promise<unknown>
/** This will be ran during any time a package is uninstalled, for example during a update
* this will be called.
*/
@@ -437,6 +434,10 @@ export type Effects = {
value: ExtractStore
}): Promise<void>
}
/** sets the version that this service's data has been migrated to */
setDataVersion(options: { version: string }): Promise<void>
/** returns the version that this service's data has been migrated to */
getDataVersion(): Promise<string | null>
// system

View File

@@ -72,6 +72,23 @@ export class Overlay implements ExecSpawnable {
return new Overlay(effects, id, rootfs, guid)
}
static async with<T>(
effects: T.Effects,
image: { id: T.ImageId; sharedRun?: boolean },
mounts: { options: MountOptions; path: string }[],
fn: (overlay: Overlay) => Promise<T>,
): Promise<T> {
const overlay = await Overlay.of(effects, image)
try {
for (let mount of mounts) {
await overlay.mount(mount.options, mount.path)
}
return await fn(overlay)
} finally {
await overlay.destroy()
}
}
async mount(options: MountOptions, path: string): Promise<Overlay> {
path = path.startsWith("/")
? `${this.rootfs}${path}`

244
sdk/lib/util/graph.ts Normal file
View File

@@ -0,0 +1,244 @@
import { boolean } from "ts-matches"
export type Vertex<VMetadata = void, EMetadata = void> = {
metadata: VMetadata
edges: Array<Edge<EMetadata, VMetadata>>
}
export type Edge<EMetadata = void, VMetadata = void> = {
metadata: EMetadata
from: Vertex<VMetadata, EMetadata>
to: Vertex<VMetadata, EMetadata>
}
export class Graph<VMetadata = void, EMetadata = void> {
private readonly vertices: Array<Vertex<VMetadata, EMetadata>> = []
constructor() {}
addVertex(
metadata: VMetadata,
fromEdges: Array<Omit<Edge<EMetadata, VMetadata>, "to">>,
toEdges: Array<Omit<Edge<EMetadata, VMetadata>, "from">>,
): Vertex<VMetadata, EMetadata> {
const vertex: Vertex<VMetadata, EMetadata> = {
metadata,
edges: [],
}
for (let edge of fromEdges) {
const vEdge = {
metadata: edge.metadata,
from: edge.from,
to: vertex,
}
edge.from.edges.push(vEdge)
vertex.edges.push(vEdge)
}
for (let edge of toEdges) {
const vEdge = {
metadata: edge.metadata,
from: vertex,
to: edge.to,
}
edge.to.edges.push(vEdge)
vertex.edges.push(vEdge)
}
this.vertices.push(vertex)
return vertex
}
findVertex(
predicate: (vertex: Vertex<VMetadata, EMetadata>) => boolean,
): Generator<Vertex<VMetadata, EMetadata>, void> {
const veritces = this.vertices
function* gen() {
for (let vertex of veritces) {
if (predicate(vertex)) {
yield vertex
}
}
}
return gen()
}
addEdge(
metadata: EMetadata,
from: Vertex<VMetadata, EMetadata>,
to: Vertex<VMetadata, EMetadata>,
): Edge<EMetadata, VMetadata> {
const edge = {
metadata,
from,
to,
}
edge.from.edges.push(edge)
edge.to.edges.push(edge)
return edge
}
breadthFirstSearch(
from:
| Vertex<VMetadata, EMetadata>
| ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
): Generator<Vertex<VMetadata, EMetadata>, void> {
const visited: Array<Vertex<VMetadata, EMetadata>> = []
function* rec(
vertex: Vertex<VMetadata, EMetadata>,
): Generator<Vertex<VMetadata, EMetadata>, void> {
if (visited.includes(vertex)) {
return
}
visited.push(vertex)
yield vertex
let generators = vertex.edges
.filter((e) => e.from === vertex)
.map((e) => rec(e.to))
while (generators.length) {
let prev = generators
generators = []
for (let gen of prev) {
const next = gen.next()
if (!next.done) {
generators.push(gen)
yield next.value
}
}
}
}
if (from instanceof Function) {
let generators = this.vertices.filter(from).map(rec)
return (function* () {
while (generators.length) {
let prev = generators
generators = []
for (let gen of prev) {
const next = gen.next()
if (!next.done) {
generators.push(gen)
yield next.value
}
}
}
})()
} else {
return rec(from)
}
}
reverseBreadthFirstSearch(
to:
| Vertex<VMetadata, EMetadata>
| ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
): Generator<Vertex<VMetadata, EMetadata>, void> {
const visited: Array<Vertex<VMetadata, EMetadata>> = []
function* rec(
vertex: Vertex<VMetadata, EMetadata>,
): Generator<Vertex<VMetadata, EMetadata>, void> {
if (visited.includes(vertex)) {
return
}
visited.push(vertex)
yield vertex
let generators = vertex.edges
.filter((e) => e.to === vertex)
.map((e) => rec(e.from))
while (generators.length) {
let prev = generators
generators = []
for (let gen of prev) {
const next = gen.next()
if (!next.done) {
generators.push(gen)
yield next.value
}
}
}
}
if (to instanceof Function) {
let generators = this.vertices.filter(to).map(rec)
return (function* () {
while (generators.length) {
let prev = generators
generators = []
for (let gen of prev) {
const next = gen.next()
if (!next.done) {
generators.push(gen)
yield next.value
}
}
}
})()
} else {
return rec(to)
}
}
shortestPath(
from:
| Vertex<VMetadata, EMetadata>
| ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
to:
| Vertex<VMetadata, EMetadata>
| ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
): Array<Edge<EMetadata, VMetadata>> | void {
const isDone =
to instanceof Function
? to
: (v: Vertex<VMetadata, EMetadata>) => v === to
const path: Array<Edge<EMetadata, VMetadata>> = []
const visited: Array<Vertex<VMetadata, EMetadata>> = []
function* check(
vertex: Vertex<VMetadata, EMetadata>,
path: Array<Edge<EMetadata, VMetadata>>,
): Generator<undefined, Array<Edge<EMetadata, VMetadata>> | undefined> {
if (isDone(vertex)) {
return path
}
if (visited.includes(vertex)) {
return
}
visited.push(vertex)
yield
let generators = vertex.edges
.filter((e) => e.from === vertex)
.map((e) => check(e.to, [...path, e]))
while (generators.length) {
let prev = generators
generators = []
for (let gen of prev) {
const next = gen.next()
if (next.done === true) {
if (next.value) {
return next.value
}
} else {
generators.push(gen)
yield
}
}
}
}
if (from instanceof Function) {
let generators = this.vertices.filter(from).map((v) => check(v, []))
while (generators.length) {
let prev = generators
generators = []
for (let gen of prev) {
const next = gen.next()
if (next.done === true) {
if (next.value) {
return next.value
}
} else {
generators.push(gen)
}
}
}
} else {
const gen = check(from, [])
while (true) {
const next = gen.next()
if (next.done) {
return next.value
}
}
}
}
}

View File

@@ -0,0 +1,78 @@
import { ValidateExVer } from "../exver"
import * as T from "../types"
export const IMPOSSIBLE = Symbol("IMPOSSIBLE")
export type VersionOptions<Version extends string> = {
/** The version being described */
version: Version & ValidateExVer<Version>
/** The release notes for this version */
releaseNotes: string
/** Data migrations for this version */
migrations: {
/**
* A migration from the previous version
* Leave blank to indicate no migration is necessary
* Set to `IMPOSSIBLE` to indicate migrating from the previous version is not possible
*/
up?: ((opts: { effects: T.Effects }) => Promise<void>) | typeof IMPOSSIBLE
/**
* A migration to the previous version
* Leave blank to indicate no migration is necessary
* Set to `IMPOSSIBLE` to indicate downgrades are prohibited
*/
down?: ((opts: { effects: T.Effects }) => Promise<void>) | typeof IMPOSSIBLE
/**
* Additional migrations, such as fast-forward migrations, or migrations from other flavors
*/
other?: Record<string, (opts: { effects: T.Effects }) => Promise<void>>
}
}
export class VersionInfo<Version extends string> {
private _version: null | Version = null
private constructor(
readonly options: VersionOptions<Version> & { satisfies: string[] },
) {}
static of<Version extends string>(options: VersionOptions<Version>) {
return new VersionInfo<Version>({ ...options, satisfies: [] })
}
/** Specify a version that this version is 100% backwards compatible to */
satisfies<V extends string>(
version: V & ValidateExVer<V>,
): VersionInfo<Version> {
return new VersionInfo({
...this.options,
satisfies: [...this.options.satisfies, version],
})
}
}
function __type_tests() {
const version: VersionInfo<"1.0.0:0"> = VersionInfo.of({
version: "1.0.0:0",
releaseNotes: "",
migrations: {},
})
.satisfies("#other:1.0.0:0")
.satisfies("#other:2.0.0:0")
// @ts-expect-error
.satisfies("#other:2.f.0:0")
let a: VersionInfo<"1.0.0:0"> = version
// @ts-expect-error
let b: VersionInfo<"1.0.0:3"> = version
VersionInfo.of({
// @ts-expect-error
version: "test",
releaseNotes: "",
migrations: {},
})
VersionInfo.of({
// @ts-expect-error
version: "test" as string,
releaseNotes: "",
migrations: {},
})
}

View File

@@ -0,0 +1,210 @@
import { ExtendedVersion, VersionRange } from "../exver"
import * as T from "../types"
import { Graph, Vertex } from "../util/graph"
import { once } from "../util/once"
import { IMPOSSIBLE, VersionInfo } from "./VersionInfo"
export class VersionGraph<CurrentVersion extends string> {
private readonly graph: () => Graph<
ExtendedVersion | VersionRange,
((opts: { effects: T.Effects }) => Promise<void>) | undefined
>
private constructor(
readonly current: VersionInfo<CurrentVersion>,
versions: Array<VersionInfo<any>>,
) {
this.graph = once(() => {
const graph = new Graph<
ExtendedVersion | VersionRange,
((opts: { effects: T.Effects }) => Promise<void>) | undefined
>()
const flavorMap: Record<
string,
[
ExtendedVersion,
VersionInfo<any>,
Vertex<
ExtendedVersion | VersionRange,
((opts: { effects: T.Effects }) => Promise<void>) | undefined
>,
][]
> = {}
for (let version of [current, ...versions]) {
const v = ExtendedVersion.parse(version.options.version)
const vertex = graph.addVertex(v, [], [])
const flavor = v.flavor || ""
if (!flavorMap[flavor]) {
flavorMap[flavor] = []
}
flavorMap[flavor].push([v, version, vertex])
}
for (let flavor in flavorMap) {
flavorMap[flavor].sort((a, b) => a[0].compareForSort(b[0]))
let prev:
| [
ExtendedVersion,
VersionInfo<any>,
Vertex<
ExtendedVersion | VersionRange,
(opts: { effects: T.Effects }) => Promise<void>
>,
]
| undefined = undefined
for (let [v, version, vertex] of flavorMap[flavor]) {
if (version.options.migrations.up !== IMPOSSIBLE) {
let range
if (prev) {
graph.addEdge(version.options.migrations.up, prev[2], vertex)
range = VersionRange.anchor(">=", prev[0]).and(
VersionRange.anchor("<", v),
)
} else {
range = VersionRange.anchor("<", v)
}
const vRange = graph.addVertex(range, [], [])
graph.addEdge(version.options.migrations.up, vRange, vertex)
}
if (version.options.migrations.down !== IMPOSSIBLE) {
let range
if (prev) {
graph.addEdge(version.options.migrations.down, vertex, prev[2])
range = VersionRange.anchor(">=", prev[0]).and(
VersionRange.anchor("<", v),
)
} else {
range = VersionRange.anchor("<", v)
}
const vRange = graph.addVertex(range, [], [])
graph.addEdge(version.options.migrations.down, vertex, vRange)
}
if (version.options.migrations.other) {
for (let rangeStr in version.options.migrations.other) {
const range = VersionRange.parse(rangeStr)
const vRange = graph.addVertex(range, [], [])
graph.addEdge(
version.options.migrations.other[rangeStr],
vRange,
vertex,
)
for (let matching of graph.findVertex(
(v) =>
v.metadata instanceof ExtendedVersion &&
v.metadata.satisfies(range),
)) {
graph.addEdge(
version.options.migrations.other[rangeStr],
matching,
vertex,
)
}
}
}
}
}
return graph
})
}
currentVersion = once(() =>
ExtendedVersion.parse(this.current.options.version),
)
static of<
CurrentVersion extends string,
OtherVersions extends Array<VersionInfo<any>>,
>(
currentVersion: VersionInfo<CurrentVersion>,
...other: EnsureUniqueId<OtherVersions, OtherVersions, CurrentVersion>
) {
return new VersionGraph(currentVersion, other as Array<VersionInfo<any>>)
}
async migrate({
effects,
from,
to,
}: {
effects: T.Effects
from: ExtendedVersion
to: ExtendedVersion
}) {
const graph = this.graph()
if (from && to) {
const path = graph.shortestPath(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(from)) ||
(v.metadata instanceof ExtendedVersion && v.metadata.equals(from)),
(v) =>
(v.metadata instanceof VersionRange && v.metadata.satisfiedBy(to)) ||
(v.metadata instanceof ExtendedVersion && v.metadata.equals(to)),
)
if (path) {
for (let edge of path) {
if (edge.metadata) {
await edge.metadata({ effects })
}
await effects.setDataVersion({ version: edge.to.metadata.toString() })
}
return
}
}
throw new Error()
}
canMigrateFrom = once(() =>
Array.from(
this.graph().reverseBreadthFirstSearch(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(this.currentVersion())) ||
(v.metadata instanceof ExtendedVersion &&
v.metadata.equals(this.currentVersion())),
),
).reduce(
(acc, x) =>
acc.or(
x.metadata instanceof VersionRange
? x.metadata
: VersionRange.anchor("=", x.metadata),
),
VersionRange.none(),
),
)
canMigrateTo = once(() =>
Array.from(
this.graph().breadthFirstSearch(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(this.currentVersion())) ||
(v.metadata instanceof ExtendedVersion &&
v.metadata.equals(this.currentVersion())),
),
).reduce(
(acc, x) =>
acc.or(
x.metadata instanceof VersionRange
? x.metadata
: VersionRange.anchor("=", x.metadata),
),
VersionRange.none(),
),
)
}
export function setupVersionGraph<
CurrentVersion extends string,
OtherVersions extends Array<VersionInfo<any>>,
>(
current: VersionInfo<CurrentVersion>,
...other: EnsureUniqueId<OtherVersions, OtherVersions, CurrentVersion>
) {
return VersionGraph.of<CurrentVersion, OtherVersions>(current, ...other)
}
// prettier-ignore
export type EnsureUniqueId<A, B = A, OtherVersions = never> =
B extends [] ? A :
B extends [VersionInfo<infer Version>, ...infer Rest] ? (
Version extends OtherVersions ? "One or more versions are not unique"[] :
EnsureUniqueId<A, Rest, Version | OtherVersions>
) : "There exists a migration that is not a Migration"[]

View File

@@ -88,6 +88,8 @@ export module Mock {
title: 'Bitcoin Core',
version: '0.21.0:0',
satisfies: [],
canMigrateTo: '!',
canMigrateFrom: '*',
gitHash: 'abcdefgh',
description: {
short: 'A Bitcoin full node by Bitcoin Core.',
@@ -132,6 +134,8 @@ export module Mock {
title: 'Lightning Network Daemon',
version: '0.11.1:0',
satisfies: [],
canMigrateTo: '!',
canMigrateFrom: '*',
gitHash: 'abcdefgh',
description: {
short: 'A bolt spec compliant client.',
@@ -188,6 +192,8 @@ export module Mock {
title: 'Bitcoin Proxy',
version: '0.2.2:0',
satisfies: [],
canMigrateTo: '!',
canMigrateFrom: '*',
gitHash: 'lmnopqrx',
description: {
short: 'A super charger for your Bitcoin node.',
@@ -1684,6 +1690,7 @@ export module Mock {
state: 'installed',
manifest: MockManifestBitcoind,
},
dataVersion: MockManifestBitcoind.version,
icon: '/assets/img/service-icons/bitcoind.svg',
lastBackup: null,
status: {
@@ -1860,6 +1867,7 @@ export module Mock {
state: 'installed',
manifest: MockManifestBitcoinProxy,
},
dataVersion: MockManifestBitcoinProxy.version,
icon: '/assets/img/service-icons/btc-rpc-proxy.png',
lastBackup: null,
status: {
@@ -1908,6 +1916,7 @@ export module Mock {
state: 'installed',
manifest: MockManifestLnd,
},
dataVersion: MockManifestLnd.version,
icon: '/assets/img/service-icons/lnd.png',
lastBackup: null,
status: {

View File

@@ -91,6 +91,7 @@ export const mockPatchData: DataModel = {
version: '0.20.0:0',
},
},
dataVersion: '0.20.0:0',
icon: '/assets/img/service-icons/bitcoind.svg',
lastBackup: null,
status: {
@@ -295,6 +296,7 @@ export const mockPatchData: DataModel = {
version: '0.11.0:0.0.1',
},
},
dataVersion: '0.11.0:0.0.1',
icon: '/assets/img/service-icons/lnd.png',
lastBackup: null,
status: {