Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

@flight-control/devops-utils

Package Overview
Dependencies
Maintainers
1
Versions
13
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@flight-control/devops-utils - npm Package Compare versions

Comparing version 0.8.16 to 0.8.17

44

dist/lib-utils/fleetctl-utils/flightfile-sanity-check.js

@@ -10,2 +10,5 @@ "use strict";

const chalk_1 = __importDefault(require("chalk"));
function log_feck(...args) {
console.warn(...args.map((v) => chalk_1.default.red.bold(v.toString())));
}
function sanity_check_flightfile(flightfile_root) {

@@ -16,32 +19,39 @@ const fleetfile = flightfile_utils_1.load_flightfile(flightfile_root);

});
const expectation = JSON.parse(expectation_doc);
const tier_specific_conf = flightfile_utils_1.select_tier(fleetfile, fleetfile.operative.tier_id);
const tier_expectated_conf_parts = JSON.parse(expectation_doc); // NOTE: This is neither spec nor conf — but we can't rely on anything in it, so... conf...
const tier_established_spec = flightfile_utils_1.select_tier(fleetfile, fleetfile.operative.tier_id);
// const established_lut = idmap_from_list(tier_specific_conf.services, "name")
// const expected_lut = idmap_from_list(expectation.services, "name")
const established_lut = tier_specific_conf.services;
const expected_lut = expectation.services;
functional_utils_1.keys(expected_lut).map(squad_id => {
const exp_squad = expected_lut[squad_id];
const est_squad = established_lut[squad_id];
const established_service_squad_lut = tier_established_spec.services;
const expected_service_squad_lut = tier_expectated_conf_parts.services;
if (!expected_service_squad_lut) {
return log_feck("no services (unit-squads) defined in expectation");
}
if (!established_service_squad_lut) {
return log_feck("no services (unit-squads) defined in expectation");
}
functional_utils_1.keys(expected_service_squad_lut).map(squad_id => {
const exp_squad = expected_service_squad_lut[squad_id];
const est_squad = established_service_squad_lut[squad_id];
if (!est_squad) {
console.warn(chalk_1.default.red.bold(`squad "${squad_id}" is missing in Flightfile. Example:\n ${chalk_1.default.white(JSON.stringify({
log_feck(`squad "${squad_id}" is missing in Flightfile. Example:\n ${chalk_1.default.white(JSON.stringify({
[squad_id]: exp_squad
}, undefined, 4))}`));
}, undefined, 4))}`);
}
else {
functional_utils_1.keys(exp_squad).map(unit_id => {
const exp_item = exp_squad[unit_id];
const est_item = est_squad[unit_id];
if (!est_item || !exp_item) {
console.warn(chalk_1.default.red.bold(`service-unit "${unit_id}" is missing in Flightfile. Example:\n ${chalk_1.default.white(JSON.stringify({
const expected_unit_spec = exp_squad[unit_id];
const established_unit_conf = est_squad[unit_id];
if (!established_unit_conf || !expected_unit_spec) {
log_feck(`service-unit "${unit_id}" is missing in Flightfile. Example:\n ${chalk_1.default.white(JSON.stringify({
[squad_id]: {
[unit_id]: exp_squad[unit_id]
}
}, undefined, 4))}`));
}, undefined, 4))}`);
}
else {
functional_utils_1.keys(exp_item).map(prop_str => {
functional_utils_1.keys(expected_unit_spec).map(prop_str => {
const prop = prop_str;
if (exp_item[prop] !== est_item[prop]) {
console.warn(chalk_1.default.red.bold(`property "${prop}" of entity "${unit_id}" mismatch expectation (expected: "${exp_item[prop]}", found: "${est_item[prop]}")`));
if (expected_unit_spec[prop] !==
established_unit_conf[prop]) {
log_feck(`property "${prop}" of entity "${unit_id}" mismatch expectation (expected: "${expected_unit_spec[prop]}", found: "${established_unit_conf[prop]}")`);
}

@@ -48,0 +58,0 @@ });

86

dist/lib-utils/fleetctl-utils/flightfile-types.d.ts
import { JsonHash, Str, Li, IdMap, Nat, Int, ChronWallUtc } from "../fun-utils/onyx-std-types";
export declare type FleetTierConf = Partial<FleetTierAlt<IdMap<IdMap<Partial<FlightCraft>>>>>;
export declare type FleetOpsOperativeConf = Partial<FleetOpsOperativeSpecBase>;
declare type FleetTierKind = "dev" | "test" | "stage" | "prod" | "replicator";
declare type FleetTierContext = "ops_only" | "build_only" | "all";
export declare type FleetOpsBase = {
export declare type FleetfileBaseConf = FleetOperativeEntry_Mixin & FleetTierConf;
export declare type FleetOpsOperativeConf = Partial<FleetOpsOperativeSpec_Core>;
export declare type FleetOpsCentral = FleetOperativeEntry_Mixin & {
tiers: IdMap<FleetTierSpec>;
};
export declare type FleetOperativeEntry_Mixin = {
operative: FleetOpsOperativeSpec;
tiers: IdMap<FleetTier>;
};
export declare type FleetOpsOperativeSpec = FleetOpsOperativeSpecBase & FleetOpsOperativeSpecInternal;
export declare type FleetOpsOperativeSpecBase = {
export declare type FleetOpsOperativeSpec = FleetOpsOperativeSpec_Core & FleetOpsOperativeSpec_Intl_;
export declare type FleetOpsOperativeSpec_Core = {
fleet_name: Str;
dir: Str;
tier_id: Str;
repos: RepoTypes;
repos: ReposCollections_Mixin;
};
declare type FleetOpsOperativeSpecInternal = {
declare type FleetOpsOperativeSpec_Intl_ = {
_snapshot_details?: {

@@ -24,13 +24,11 @@ tier_id: Str;

};
export declare type FleetTier = FleetTierAlt<IdMap<IdMap<FlightCraft>>>;
declare type RepoTypes = {
tiers: IdMap<RepoConf>;
sources: IdMap<RepoConf>;
};
declare type FleetTierAlt<ST> = FleetTierInternal & {
export declare type FleetTierConf = Partial<FleetTier_Base<FlightUnitConf, SshCfg>>;
export declare type FleetTierSpec = FleetTier_Base<FlightUnitSpec, SshSpec> & FleetTier_Intl_;
export declare type FleetTier__Mythical_To_Separate = FleetTierConf;
declare type FleetTier_Base<SquadT, SshT> = {
fleet_name: Str;
spec: {
tier_id: Str;
tier_role: FleetTierKind;
tier_context: FleetTierContext;
tier_role: FleetTierRole_Tag;
tier_context: FleetTierContext_Tag;
paths_deprecated_?: {

@@ -43,9 +41,18 @@ log_root: Str;

};
ssh_access?: SshCfg;
ssh_access?: SshT;
backup_conf: BackupConf;
};
repos?: RepoTypes;
services: ST;
repos?: ReposCollections_Mixin;
services: IdMap<IdMap<SquadT>>;
confs: IdMap<JsonHash>;
};
declare type FleetTierRole_Tag = "dev" | "test" | "stage" | "prod" | "replicator";
declare type FleetTierContext_Tag = "ops_only" | "build_only" | "all";
declare type FleetTier_Intl_ = {
unit_lut_: IdMap<FlightUnitSpec>;
};
declare type ReposCollections_Mixin = {
tiers: IdMap<RepoConf>;
sources: IdMap<RepoConf>;
};
export declare type SshCfg = {

@@ -55,6 +62,7 @@ host: Str;

user: Str;
secret_file?: Str;
secret_filename?: Str;
};
declare type FleetTierInternal = {
unit_lut_: IdMap<FlightCraft>;
export declare type SshSpec = Required<SshCfg> & Ssh_Intl_;
declare type Ssh_Intl_ = {
secret_path_: Str;
};

@@ -77,3 +85,3 @@ export declare type BackupConf = {

declare type BackupScheduledAction = {
action: BackupActions;
action: BackupAction_Tag;
tier: Str;

@@ -83,3 +91,3 @@ at?: ChronWallUtc;

};
declare type BackupActions = "snapshot" | "replicate" | "rotate";
declare type BackupAction_Tag = "snapshot" | "replicate" | "rotate";
declare type WeekdayNumber = 1 | 2 | 3 | 4 | 5 | 6 | 7;

@@ -94,3 +102,6 @@ export declare type RepoConf = {

};
declare type FlightCraftInternalFields = {
export declare type FlightUnitAny = FlightUnitGeneric | FlightUnitDb | FlightUnitFs;
export declare type FlightUnitSpec = FlightUnit_Intl_ & FlightUnitAny;
export declare type FlightUnitConf = Partial<FlightUnitAny>;
declare type FlightUnit_Intl_ = {
group: Str;

@@ -100,21 +111,20 @@ name: Str;

};
declare type FlightCraftBase = FlightCraftInternalFields & {
declare type FlightUnit_Core = {
deps?: IdMap<Str | JsonHash>;
repo?: Str;
};
export declare type FlightCraftDb = FlightCraftBase & {
export declare type FlightUnitDb = FlightUnit_Core & {
type: "db";
kind: FlightCraftDb_Kind;
kind: FlightUnitDb_KindTag;
};
export declare type FlightCraftFs = FlightCraftBase & {
export declare type FlightUnitFs = FlightUnit_Core & {
type: "fs";
kind: FlightCraftFs_Kind;
kind: FlightUnitFs_Kind;
};
export declare type FlightCraftGeneric = FlightCraftBase & {
type?: FlightCraftType;
export declare type FlightUnitGeneric = FlightUnit_Core & {
type?: FlightUnit_TypeTag;
};
export declare type FlightCraftType = "process" | "external" | "db" | "fs";
export declare type FlightCraftDb_Kind = "pg" | "mongo" | "redis" | "elastic" | "cassandra";
export declare type FlightCraftFs_Kind = "private" | "public";
export declare type FlightCraft = FlightCraftGeneric | FlightCraftDb | FlightCraftFs;
export declare type FlightUnit_TypeTag = "process" | "external" | "db" | "fs";
export declare type FlightUnitDb_KindTag = "pg" | "mongo" | "redis" | "elastic" | "cassandra";
export declare type FlightUnitFs_Kind = "private" | "public";
export {};

39

dist/lib-utils/fleetctl-utils/flightfile-utils.d.ts

@@ -1,3 +0,3 @@

import { Str, Undef, IdMap, Li, JsonHash, Bool } from "../fun-utils/onyx-std-types";
import { FleetTier, FlightCraft, FleetOpsBase } from "./flightfile-types";
import { Str, Undef, Li, Bool } from "../fun-utils/onyx-std-types";
import { FlightUnitSpec, FleetOpsCentral, FleetTierSpec } from "./flightfile-types";
import { HashMatchPattern } from "../fun-utils/functional-utils";

@@ -7,33 +7,10 @@ export declare function specify_conf<T extends {

}>(conf: T): T;
export declare function load_flightfile(flightfile_path_: string, verbose_inspection?: Str | Undef): FleetOpsBase;
export declare function flightfile_from_first_arg(): FleetOpsBase;
export declare function select_tier(fleetfile: FleetOpsBase, tier_id: Str): {
unit_lut_: IdMap<FlightCraft>;
} & {
fleet_name: string;
spec: {
tier_id: string;
tier_role: "dev" | "test" | "stage" | "prod" | "replicator";
tier_context: "ops_only" | "build_only" | "all";
paths_deprecated_?: {
log_root: string;
} | undefined;
operative_user?: {
username: string;
secret?: string | undefined;
} | undefined;
ssh_access?: import("./flightfile-types").SshCfg | undefined;
backup_conf: import("./flightfile-types").BackupConf;
};
repos?: {
tiers: IdMap<import("./flightfile-types").RepoConf>;
sources: IdMap<import("./flightfile-types").RepoConf>;
} | undefined;
services: IdMap<IdMap<FlightCraft>>;
confs: IdMap<JsonHash>;
};
export declare function select_unit(flight_tier_conf: FleetTier, unit_name: Str): FlightCraft | undefined;
export declare function select_units(tier: FleetTier, hash_pattern: HashMatchPattern<FlightCraft>): Li<FlightCraft>;
export declare function load_flightfile(flightfile_path_: string, verbose_inspection?: Str | Undef): FleetOpsCentral;
export declare function load_flightfile__(flightfile_path_: string, verbose_inspection?: Str | Undef): FleetOpsCentral;
export declare function flightfile_from_first_arg(): FleetOpsCentral;
export declare function select_tier(fleetfile: FleetOpsCentral, tier_id: Str): FleetTierSpec;
export declare function select_unit(flight_tier_conf: FleetTierSpec, unit_name: Str): FlightUnitSpec | undefined;
export declare function select_units(tier: FleetTierSpec, hash_pattern: HashMatchPattern<FlightUnitSpec>): Li<FlightUnitSpec>;
export declare function is_fleetbase_bootstrapped_and_ready(): Bool;
export declare function derive_operative_owner_combo(): Str;
export declare function ensure_operative_fs_ownership(): void;

269

dist/lib-utils/fleetctl-utils/flightfile-utils.js

@@ -18,2 +18,3 @@ "use strict";

const build_utils_1 = require("../sh-utils/build-utils");
const ssh_utils_1 = require("../sh-utils/ssh-utils");
function specify_conf(conf) {

@@ -29,121 +30,162 @@ // are we — as a conf-file — being _executed_? Then the user wants conf-data

function load_flightfile(flightfile_path_, verbose_inspection = undefined) {
try {
return load_flightfile__(flightfile_path_, verbose_inspection);
}
catch (err1) {
const entrypoint_flightfile_path = Path.normalize(flightfile_path_);
const flightdir = Path.dirname(entrypoint_flightfile_path);
throw cli_utils_1.die("Couldn't compose Flightfile for path:", flightdir, err1);
}
}
exports.load_flightfile = load_flightfile;
function load_flightfile__(flightfile_path_, verbose_inspection = undefined) {
const entrypoint_flightfile_path = Path.normalize(flightfile_path_);
const flightdir = Path.dirname(entrypoint_flightfile_path);
try {
verbose_inspection &&
cli_utils_1.say("Load all required files for full composite Flightfile");
ensure_all_conf_files_compiled();
const tier_specs_path = Path.join(flightdir, "tiers.d");
const available_tier_specs = (() => {
try {
return file_utils_1.readdir(tier_specs_path);
verbose_inspection &&
cli_utils_1.say("Load all required files for full composite Flightfile");
ensure_all_conf_files_compiled();
const tier_specs_path = Path.join(flightdir, "tiers.d");
const available_tier_specs = (() => {
try {
return file_utils_1.readdir(tier_specs_path);
}
catch (e) {
if (false === is_fleetbase_bootstrapped_and_ready()) {
return [];
}
catch (e) {
if (false === is_fleetbase_bootstrapped_and_ready()) {
return [];
}
else {
cli_utils_1.log_err("caught missing tiers.d repos...", process.argv);
throw e;
}
else {
cli_utils_1.log_err("caught missing tiers.d repos...", process.argv);
throw e;
}
}
})();
const maybe_local_overrides_conf = {
operative: {},
...functional_utils_1.try_else(() => file_utils_1.read_struct_file(entrypoint_flightfile_path.replace(/(\.\w+)$/, ".local.$1")), {})
};
const standard_base_conf = file_utils_1.read_struct_file(entrypoint_flightfile_path);
const operative_tier_id = os.hostname();
const default_operative_spec = {
fleet_name: standard_base_conf.fleet_name,
dir: flightdir,
// TODO should be OPERATIVE_tier_id here too - the prop
tier_id: operative_tier_id,
repos: standard_base_conf.repos // TODO should ofc be orderly and actually specify this under "operative" prop sub conf
};
const base_conf = depth_two_merge(standard_base_conf, maybe_local_overrides_conf);
verbose_inspection && cli_utils_1.say("Base Op Flightfiles");
verbose_inspection && cli_utils_1.say(JSON.stringify(base_conf, undefined, 4));
const tier_specs = functional_utils_1.fold(available_tier_specs, {}, (tier_lut_acc, tier_repo_slug) => {
const tier_specific_flightfile_path = Path.join(flightdir, "tiers.d", tier_repo_slug, "Flightsecrets");
const loaded = {
...{ confs: {} },
...file_utils_1.read_struct_file(tier_specific_flightfile_path)
};
if (!loaded.spec) {
throw Error(`Fleetsecrets conf from tier-dir "${tier_repo_slug}" is missing "spec"-conf`);
}
if (loaded.spec.ssh_access) {
loaded.spec.ssh_access = ssh_utils_1.ssh_spec_from_cfg(tier_repo_slug, loaded.spec.ssh_access);
}
const tier_id = loaded.spec.tier_id;
const verlog = verbose_inspection === tier_id ? cli_utils_1.say : cli_utils_1.noop;
verlog(`Reads tier-conf "${tier_specific_flightfile_path}" for tier-id "${tier_id}"`);
verlog(JSON.stringify(loaded, undefined, 4));
// NOTE: we allow multiple tier-definition repos, to separate
// for instance backup-replication-access and actual full
// secrets — and some fleetoperatives may have access to both /
// multiple — so we merge according to a simple two-prop-depth
// rule that promotes properties from full-confs over
// ops-only-confs
const tier_conf_pre_merge = (() => {
// TODO — since we massage the conf as we
// consolidate it into spec — shuffling around the massaged
// one as layer_b could cause information-loss?
// First get a pitch of _all_ available moving parts, sort
// them as needed, THEN do the layer-ordered-merge
// NOTE: this is not a common usage-pattern. Fix later.
const existing_part = tier_lut_acc[tier_id];
if (!existing_part)
return loaded;
verlog(`Encountered a merge-situation for tier-id "${tier_id}"`);
const [layer_a, layer_b, verbose_msg] = loaded.spec.tier_context === "all"
? [
existing_part,
loaded,
`Currently processed takes property-precedence over previously loaded tier-conf for tier-id "${tier_id}"`
]
: [
loaded,
existing_part,
`Previously loaded takes property-precedence over currently processed tier-conf for tier-id "${tier_id}"`
];
verlog(verbose_msg);
// const merge_ret = { ...layer_a } as any
// keys(layer_b).map(k => {
// merge_ret[k] = {
// ...(merge_ret[k] || {}),
// ...(layer_b as Any)[k]
// }
// })
const merge_ret = depth_two_merge(layer_a, layer_b);
return merge_ret;
})();
const base_conf = file_utils_1.read_struct_file(entrypoint_flightfile_path);
verbose_inspection && cli_utils_1.say("Base Flightfile");
verbose_inspection && cli_utils_1.say(JSON.stringify(base_conf, undefined, 4));
const operative_tier_id = os.hostname();
const operative_spec = {
fleet_name: base_conf.fleet_name,
dir: flightdir,
tier_id: operative_tier_id,
repos: base_conf.repos
const merged_tier_conf = {
...base_conf,
...tier_conf_pre_merge,
...{ unit_lut_: {} }
};
const tier_specs = functional_utils_1.fold(available_tier_specs, {}, (tier_lut_acc, tier_repo_slug) => {
const tier_specific_flightfile_path = Path.join(flightdir, "tiers.d", tier_repo_slug, "Flightsecrets");
const loaded = {
...{ confs: {} },
...file_utils_1.read_struct_file(tier_specific_flightfile_path)
};
const tier_id = loaded.spec.tier_id;
const verlog = verbose_inspection === tier_id ? cli_utils_1.say : cli_utils_1.noop;
verlog(`Reads tier-conf "${tier_specific_flightfile_path}" for tier-id "${tier_id}"`);
verlog(JSON.stringify(loaded, undefined, 4));
// NOTE: we allow multiple tier-definition repos, to separate
// for instance backup-replication-access and actual full
// secrets — and some fleetoperatives may have access to both /
// multiple — so we merge according to a simple two-prop-depth
// rule that promotes properties from full-confs over
// ops-only-confs
const tier_conf_pre_merge = (() => {
const curr = tier_lut_acc[tier_id];
if (!curr)
return loaded;
verlog(`Encountered a merge-situation for tier-id "${tier_id}"`);
const [a, b, vermsg] = loaded.spec.tier_context === "all"
? [
curr,
loaded,
`Currently processed takes property-precedence over previously loaded tier-conf for tier-id "${tier_id}"`
]
: [
loaded,
curr,
`Previously loaded takes property-precedence over currently processed tier-conf for tier-id "${tier_id}"`
];
verlog(vermsg);
const merge_ret = { ...a };
functional_utils_1.keys(b).map(k => {
merge_ret[k] = {
...(merge_ret[k] || {}),
...b[k]
};
});
return merge_ret;
})();
const merged_tier_conf = {
...base_conf,
...tier_conf_pre_merge
};
const units_lut = (merged_tier_conf.unit_lut_ = {});
functional_utils_1.pairs(merged_tier_conf.services).map(([squad_id, units]) => {
functional_utils_1.pairs(units).map(([flightcraft_id, v]) => {
if (units_lut[flightcraft_id]) {
throw Error(`All service-units must have uniquely identifying names — prefix with squad-name to ensure it ("${flightcraft_id}").`);
}
const unit_conf = tier_conf_pre_merge.confs[flightcraft_id];
if (!unit_conf &&
tier_conf_pre_merge.spec.tier_context !== "ops_only") {
throw Error(`No conf defined for service-unit "${flightcraft_id}", specify an empty conf if none required to establish omission is deliberate.`);
}
units_lut[flightcraft_id] = units[flightcraft_id] = {
...v,
group: squad_id,
name: flightcraft_id,
conf: unit_conf
};
});
if (!merged_tier_conf.services) {
throw Error(`No collection of service squads is defined at all`);
}
if (!merged_tier_conf.spec) {
throw Error(`No tier-spec is defined for tier "${tier_id}"`);
}
const is_ops_only = merged_tier_conf.spec.tier_context === "ops_only";
if (!merged_tier_conf.confs) {
if (is_ops_only) {
merged_tier_conf.confs = {};
}
else {
throw Error(`No confs defined for service units in tier "${tier_id}"`);
}
}
const units_lut = merged_tier_conf.unit_lut_;
functional_utils_1.pairs(merged_tier_conf.services).map(([squad_id, units]) => {
functional_utils_1.pairs(units).map(([flightcraft_id, v]) => {
if (units_lut[flightcraft_id]) {
throw Error(`All service-units must have uniquely identifying names — prefix with squad-name to ensure it ("${flightcraft_id}").`);
}
const unit_conf = merged_tier_conf.confs[flightcraft_id] ||
(is_ops_only && {});
if (!unit_conf) {
throw Error(`No conf defined for service-unit "${flightcraft_id}" for tier "${tier_id}", specify an empty conf if none required to establish omission is deliberate.`);
}
units_lut[flightcraft_id] = units[flightcraft_id] = {
...v,
group: squad_id,
name: flightcraft_id,
conf: unit_conf
};
});
return {
...tier_lut_acc,
[tier_id]: merged_tier_conf
};
});
const composite_conf = {
operative: operative_spec,
tiers: tier_specs
return {
...tier_lut_acc,
[tier_id]: merged_tier_conf
};
if (verbose_inspection) {
const dump_conf = { ...composite_conf.tiers[verbose_inspection] };
delete dump_conf.unit_lut_;
delete dump_conf.confs;
cli_utils_1.say(`finally merged composite_conf`);
cli_utils_1.say(JSON.stringify(dump_conf, undefined, 4));
}
return composite_conf;
});
const composite_conf = {
operative: default_operative_spec,
tiers: tier_specs
};
if (verbose_inspection) {
const dump_conf = { ...composite_conf.tiers[verbose_inspection] };
delete dump_conf.unit_lut_;
delete dump_conf.confs;
cli_utils_1.say(`finally merged composite_conf`);
cli_utils_1.say(JSON.stringify(dump_conf, undefined, 4));
}
catch (err1) {
throw cli_utils_1.die("Couldn't compose Flightfile for path:", flightdir, err1);
}
return composite_conf;
}
exports.load_flightfile = load_flightfile;
exports.load_flightfile__ = load_flightfile__;
let loaded_conf_;

@@ -209,1 +251,12 @@ function flightfile_from_first_arg() {

exports.ensure_operative_fs_ownership = ensure_operative_fs_ownership;
function depth_two_merge(layer_a, layer_b) {
const mut_ret = {};
const all_keys = functional_utils_1.uniq([...functional_utils_1.keys(layer_a), ...functional_utils_1.keys(layer_b)]);
// Crude mutating operation here. works. to tired to bother with the clumsy
// es-syntax
all_keys.map(k => (mut_ret[k] = {
...(layer_a[k] || {}),
...(layer_b[k] || {})
}));
return mut_ret; // typeof layer_a & typeof layer_b
}

11

dist/lib-utils/sh-utils/ssh-utils.d.ts

@@ -1,9 +0,4 @@

import { SshCfg, FleetOpsBase } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { SshCfg, SshSpec } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { Str } from "../../lib-utils/fun-utils/onyx-std-types";
export declare function normalize_ssh_cfg(in_cfg: SshAccessCfg): RemoteSshAccess;
export declare function render_ssh_url(cfg: RemoteSshAccess): Str;
export declare function render_ssh_secret_filepath(fleetfile: FleetOpsBase, sourced_tier_id: Str): Str;
export declare type SshAccessCfg = SshCfg & {
remote_path?: Str;
};
export declare type RemoteSshAccess = Required<SshAccessCfg>;
export declare function ssh_spec_from_cfg(tier_repo_slug: Str, in_cfg: SshCfg): SshSpec;
export declare function render_ssh_url(cfg: SshSpec): Str;

26

dist/lib-utils/sh-utils/ssh-utils.js

@@ -7,4 +7,5 @@ "use strict";

const path_1 = __importDefault(require("path"));
const flightfile_utils_1 = require("../fleetctl-utils/flightfile-utils");
function normalize_ssh_cfg(in_cfg) {
function ssh_spec_from_cfg(tier_repo_slug, in_cfg) {
const secret_filename = in_cfg.secret_filename || "ssh-secret";
const secret_path = path_1.default.join(".", "tiers.d", tier_repo_slug, secret_filename);
return {

@@ -14,7 +15,8 @@ host: in_cfg.host,

user: in_cfg.user || "",
secret_file: in_cfg.secret_file || "",
remote_path: in_cfg.remote_path || ""
secret_filename: secret_filename,
secret_path_: secret_path
// remote_path: in_cfg.remote_path || ""
};
}
exports.normalize_ssh_cfg = normalize_ssh_cfg;
exports.ssh_spec_from_cfg = ssh_spec_from_cfg;
function render_ssh_url(cfg) {

@@ -24,6 +26,10 @@ return (cfg.user ? cfg.user + "@" : "") + cfg.host;

exports.render_ssh_url = render_ssh_url;
function render_ssh_secret_filepath(fleetfile, sourced_tier_id) {
const sourced_tier = flightfile_utils_1.select_tier(fleetfile, sourced_tier_id);
return path_1.default.join(fleetfile.operative.dir, "tiers.d", sourced_tier_id, "ssh-secret");
}
exports.render_ssh_secret_filepath = render_ssh_secret_filepath;
// export function render_ssh_secret_filepath(
// fleetfile: FleetOpsCentral,
// sourced_tier_id: Str
// ): Str {
// const sourced_tier = select_tier(fleetfile, sourced_tier_id)
// return sourced_tier.spec.ssh_access!.secret_file!
// }
// export type SshAccessSpec = SshSpec & { remote_path?: Str }
// export type RemoteSshAccess = Required<SshAccessSpec>

20

dist/tooling/fctl-ops/repo-ctl.js

@@ -50,2 +50,13 @@ "use strict";

}
function ensure_flightcontrol_freshness() {
// TODO should be done async, aight!
//
// REF OUTPUT:
// Package Current Wanted Latest Location
// @flight-control/devops-utils 0.8.15 0.8.16 0.8.16 fleetbase-wacebe
//
// if (sh(`npm outdated`).split(/\n/).any(v => v.match(/^@flight-control\/devops-utils/))) {
// sh(`npm install @flight-control/devops-utils`)
// }
}
function ensure_all_repos() {

@@ -122,7 +133,4 @@ const ops_root = fleetfile.operative.dir;

cmd_over_dirs_in_dir(["tiers.d"], [
`git add . && git commit -m 'fleetctl commit from tier-env "${fleetfile.operative.tier_id}"' && git push`
// `echo 'fctl-commit from tier-env "${get_operating_user()} ${
// fleetfile.operative.tier_id
// }"'`
//
// `git status --short | head -c1 | wc -c && if [[ $(git status --short | head -c1 | wc -c) -ne 0 ]]; then ( git add . && git commit -m 'fleetctl commit from tier-env "${
`git status --short | grep . && (( git add . && git commit -m 'fleetctl commit from tier-env "${fleetfile.operative.tier_id}"' && git push ) || exit 0) || ( echo "no changes" )`
]);

@@ -157,3 +165,3 @@ }

const path = root_path_parts.join("/");
return cli_utils_1.sh(`cd ${path} && ${cmd_parts.join(" ")}`, { do_throw: true });
return cli_utils_1.sh(`cd ${path} && (${cmd_parts.join(" ")})`, { do_throw: true });
}

@@ -160,0 +168,0 @@ function rerun_self_and_die() {

6

dist/tooling/fleet-data-ctl/backup-file-level-utils.d.ts
import { Str, Li, Undef, Nat, Void, Int, Bool } from "../../lib-utils/fun-utils/onyx-std-types";
import { FlightCraft, FleetTier } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { FlightUnitSpec, FleetTierSpec } from "../../lib-utils/fleetctl-utils/flightfile-types";
declare type BackupTimeFrame = "daily" | "weekly" | "monthly";

@@ -19,3 +19,3 @@ export declare function read_list_of_snapshot_tiers(): Li<Str>;

};
export declare function make_snapshot_unit_sub_dir(snapshot_dir: Str, unit: FlightCraft, additional_subdir_part?: Str): Str;
export declare function make_snapshot_unit_sub_dir(snapshot_dir: Str, unit: FlightUnitSpec, additional_subdir_part?: Str): Str;
export declare function extract_unit_id_from_sub_dir(sub_dir: Str): Str;

@@ -27,3 +27,3 @@ /**

export declare function validate_backup_snapshot_path(snapshot_dir: Str): Void;
export declare function ensure_correct_backups_dir_owner(tierconf: FleetTier): Void;
export declare function ensure_correct_backups_dir_owner(tierconf: FleetTierSpec): Void;
export {};

7

dist/tooling/fleet-data-ctl/backup-replicate.d.ts

@@ -1,5 +0,4 @@

import { FleetTier, FleetOpsBase } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { FleetTierSpec, FleetOpsCentral, SshSpec } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { Str, Bool } from "../../lib-utils/fun-utils/onyx-std-types";
import { RemoteSshAccess } from "../../lib-utils/sh-utils/ssh-utils";
export declare function test_remote_ssh_access(cfg: RemoteSshAccess): Bool;
export declare function replicate_remote_backups(fleetfile: FleetOpsBase, operative_tier: FleetTier, sourced_tier_id: Str): undefined;
export declare function test_remote_ssh_access(cfg: SshSpec): Bool;
export declare function replicate_remote_backups(fleetfile: FleetOpsCentral, operative_tier: FleetTierSpec, sourced_tier_id: Str): undefined;

28

dist/tooling/fleet-data-ctl/backup-replicate.js

@@ -28,3 +28,6 @@ "use strict";

const remote_cmd = "pwd";
return (functional_utils_1.try_else(() => !!cli_utils_1.sh(cli_utils_1.normalize_paths_within_cmd_string(`sshpass -p $(cat "${cfg.secret_file}") ` +
if (!file_utils_1.existsf(cfg.secret_path_)) {
throw Error(`can't find ssh secrets file "${cfg.secret_path_}"`);
}
return (functional_utils_1.try_else(() => !!cli_utils_1.sh(cli_utils_1.normalize_paths_within_cmd_string(`sshpass -p $(cat "${cfg.secret_path_}") ` +
`ssh -p ${cfg.port} ${ssh_url} ${remote_cmd}`), { do_throw: true }), false) ||

@@ -38,15 +41,16 @@ // Make sure we get some clarifiying information (that sshpass will

const sourced_tier = flightfile_utils_1.select_tier(fleetfile, sourced_tier_id);
const ssh_cfg = ssh_utils_1.normalize_ssh_cfg({
secret_file: ssh_utils_1.render_ssh_secret_filepath(fleetfile, sourced_tier_id),
...sourced_tier.spec.ssh_access
});
if (!test_remote_ssh_access(ssh_cfg)) {
return cli_utils_1.die(1, `ssh-access failure for ${ssh_cfg.host}, running as user ${os_1.default.userInfo({ encoding: "utf8" }).username}`);
// const ssh_cfg = normalize_ssh_cfg({
// // secret_path_: render_ssh_secret_filepath(fleetfile, sourced_tier_id),
// ...sourced_tier.spec.ssh_access!
// })
const ssh_spec = sourced_tier.spec.ssh_access;
if (!test_remote_ssh_access(ssh_spec)) {
return cli_utils_1.die(1, `ssh-access failure for ${ssh_spec.host}, running as user ${os_1.default.userInfo({ encoding: "utf8" }).username}`);
}
const remote_ssh_url = ssh_utils_1.render_ssh_url(ssh_cfg);
const remote_ssh_url = ssh_utils_1.render_ssh_url(ssh_spec);
const remote_dailies_path = cli_utils_1.normalize_paths_within_cmd_string(`${sourced_tier.spec.backup_conf.path}/${sourced_tier_id}/daily.d/`);
const target_dailies_path = cli_utils_1.normalize_paths_within_cmd_string(`${operative_tier.spec.backup_conf.path}/${sourced_tier_id}/daily.d/`);
cli_utils_1.sh(`mkdir -p "${target_dailies_path}"`);
const dailies = cli_utils_1.sh(`sshpass -p $(cat "${ssh_cfg.secret_file}") ` +
`ssh -p ${ssh_cfg.port} ${remote_ssh_url} ls -1 ${remote_dailies_path}`, { silent: true })
const dailies = cli_utils_1.sh(`sshpass -p $(cat "${ssh_spec.secret_path_}") ` +
`ssh -p ${ssh_spec.port} ${remote_ssh_url} ls -1 ${remote_dailies_path}`, { silent: true })
.split(/\n/)

@@ -75,4 +79,4 @@ .filter(v => !!v)

// TODO - handle connection problems - disconnected rsync => retry x t
const replicate_snapshot_cmd = cli_utils_1.normalize_paths_within_cmd_string(`sshpass -p $(cat "${ssh_cfg.secret_file}") ` +
`rsync -az ${cli_utils_1.is_dry_run() ? "-n" : ""} -e 'ssh -p ${ssh_cfg.port}' ${remote_ssh_url}:${remote_dailies_path}/${snapshot_fname}/* ${temp_snapshot_target_path}/`);
const replicate_snapshot_cmd = cli_utils_1.normalize_paths_within_cmd_string(`sshpass -p $(cat "${ssh_spec.secret_path_}") ` +
`rsync -az ${cli_utils_1.is_dry_run() ? "-n" : ""} -e 'ssh -p ${ssh_spec.port}' ${remote_ssh_url}:${remote_dailies_path}/${snapshot_fname}/* ${temp_snapshot_target_path}/`);
cli_utils_1.sh(replicate_snapshot_cmd);

@@ -79,0 +83,0 @@ backup_file_level_utils_1.move_snapshot_temp_dir_to_final_path(temp_snapshot_target_path, final_snapshot_target_path);

4

dist/tooling/fleet-data-ctl/backup-rotate-periodizations.d.ts

@@ -1,3 +0,3 @@

import { FleetOpsBase } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { FleetOpsCentral } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { Str } from "../../lib-utils/fun-utils/onyx-std-types";
export declare function rotate_snapshot_periodizations(fleetfile: FleetOpsBase, snapshot_tier_id: Str): undefined;
export declare function rotate_snapshot_periodizations(fleetfile: FleetOpsCentral, snapshot_tier_id: Str): undefined;

6

dist/tooling/fleet-data-ctl/backup-snapshot-restore.d.ts

@@ -1,4 +0,4 @@

import { FleetTier, FleetOpsBase } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { FleetTierSpec, FleetOpsCentral } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { Str, Bool, Rex } from "../../lib-utils/fun-utils/onyx-std-types";
export declare function backup_snapshot_now(fleetfile: FleetOpsBase, tier: FleetTier): void;
export declare function backup_snapshot_restore(snapshot_dir: Str, target_tier: FleetTier, unit_filter: Rex, restore_mode: "dev" | "prod", do_debug_merge: Bool, skip_missing: Bool): void;
export declare function backup_snapshot_now(fleetfile: FleetOpsCentral, tier: FleetTierSpec): void;
export declare function backup_snapshot_restore(snapshot_dir: Str, target_tier: FleetTierSpec, unit_filter: Rex, restore_mode: "dev" | "prod", do_debug_merge: Bool, skip_missing: Bool): void;

6

dist/tooling/staging-static-analysis/analysis-utils/fleet-asg-types.d.ts

@@ -5,3 +5,3 @@ import { JsonHash, Str, Li, IdMap } from "../../../lib-utils/fun-utils/onyx-std-types";

squads: IdMap<Squad>;
flightcrafts: IdMap<FlightCraft>;
flightcrafts: IdMap<FlightUnit>;
};

@@ -37,3 +37,3 @@ export declare function create_flightcraft(name: Str, filename: Str): {

readme_doc: Str;
flightcrafts: IdMap<FlightCraft>;
flightcrafts: IdMap<FlightUnit>;
databases: IdMap<DbEntry>;

@@ -45,3 +45,3 @@ };

};
export declare type FlightCraft = {
export declare type FlightUnit = {
filepath: Str;

@@ -48,0 +48,0 @@ name: Str;

8

dist/tooling/staging-static-analysis/analysis-utils/fleet-doc-asg.d.ts
import { Str, IdMap } from "../../../lib-utils/fun-utils/onyx-std-types";
import { DbEntry, Squad, FlightCraft } from "./fleet-asg-types";
import { DbEntry, Squad, FlightUnit } from "./fleet-asg-types";
export * from "./fleet-asg-types";

@@ -7,8 +7,8 @@ export declare function get_asg_root(): {

squads: IdMap<Squad>;
flightcrafts: IdMap<FlightCraft>;
flightcrafts: IdMap<FlightUnit>;
};
export declare function ensure_flightcraft(filename: Str): FlightCraft;
export declare function ensure_flightcraft(filename: Str): FlightUnit;
export declare function ensure_squad(squad_key: Str): Squad;
export declare function ensure_squad_by_filename(filename: Str): Squad;
export declare function ensure_flightcraft_by_filename(filename: Str): FlightCraft;
export declare function ensure_flightcraft_by_filename(filename: Str): FlightUnit;
export declare function is_flightcraft_main_file(filename: Str, src_text: Str): boolean;

@@ -15,0 +15,0 @@ export declare function load_squad_main_doc_readme(filename: Str): Squad;

4

dist/tooling/staging-static-analysis/generate-graph-asg.d.ts
import * as Grhl from "../../lib-utils/graphul/ghrl-main";
import { Bool } from "../../lib-utils/fun-utils/onyx-std-types";
import { FleetTier } from "../../lib-utils/fleetctl-utils/flightfile-types";
import { FleetTierSpec } from "../../lib-utils/fleetctl-utils/flightfile-types";
export declare type GraphType = "bind-plane" | "logical-plane" | "control-plane";
export declare function generate_graph(flight_conf: FleetTier, wanted_graph_type_: GraphType, DEBUG_PSEUDO_ELEMENTS: Bool): Grhl.GraphDocRoot;
export declare function generate_graph(flight_conf: FleetTierSpec, wanted_graph_type_: GraphType, DEBUG_PSEUDO_ELEMENTS: Bool): Grhl.GraphDocRoot;

2

package.json
{
"name": "@flight-control/devops-utils",
"version": "0.8.16",
"version": "0.8.17",
"description": "Dev-ops utilities to automate and simplify: staging, deployment, data-managment, dependency graphs, automated API-docs, and increase insight and graspability of a microservice eco-systems. TS-type-checking available for conf-files — or simply use `fleetctl` with a json `\"Flightfile\"",

@@ -5,0 +5,0 @@ "main": "dist/lib-all.js",

110

src/lib-utils/fleetctl-utils/flightfile-sanity-check.ts
import { say, sh } from "../sh-utils/cli-utils"
import { JsonHash, Li, Str } from "../fun-utils/onyx-std-types"
import { JsonHash, Li, Str, Any } from "../fun-utils/onyx-std-types"
import {

@@ -8,6 +8,14 @@ load_flightfile,

import { idmap_from_list, keys } from "../fun-utils/functional-utils"
import { FleetTier, FlightCraft } from "./flightfile-types"
import {
FleetTierSpec,
FlightUnitSpec,
FleetTierConf
} from "./flightfile-types"
import chalk from "chalk"
import { die } from "../sh-utils/cli-utils"
function log_feck(...args: Any) {
console.warn(...args.map((v: Any) => chalk.red.bold(v.toString())))
}
export function sanity_check_flightfile(flightfile_root: Str) {

@@ -23,4 +31,7 @@ const fleetfile = load_flightfile(flightfile_root)

const expectation = JSON.parse(expectation_doc) as FleetTier
const tier_specific_conf = select_tier(
const tier_expectated_conf_parts = JSON.parse(
expectation_doc
) as FleetTierConf // NOTE: This is neither spec nor conf — but we can't rely on anything in it, so... conf...
const tier_established_spec = select_tier(
fleetfile,

@@ -31,54 +42,59 @@ fleetfile.operative.tier_id

// const expected_lut = idmap_from_list(expectation.services, "name")
const established_lut = tier_specific_conf.services
const expected_lut = expectation.services
const established_service_squad_lut = tier_established_spec.services
const expected_service_squad_lut = tier_expectated_conf_parts.services
keys(expected_lut).map(squad_id => {
const exp_squad = expected_lut[squad_id]
const est_squad = established_lut[squad_id]
if (!expected_service_squad_lut) {
return log_feck("no services (unit-squads) defined in expectation")
}
if (!established_service_squad_lut) {
return log_feck("no services (unit-squads) defined in expectation")
}
keys(expected_service_squad_lut).map(squad_id => {
const exp_squad = expected_service_squad_lut[squad_id]
const est_squad = established_service_squad_lut[squad_id]
if (!est_squad) {
console.warn(
chalk.red.bold(
`squad "${squad_id}" is missing in Flightfile. Example:\n ${chalk.white(
JSON.stringify(
{
[squad_id]: exp_squad
},
undefined,
4
)
)}`
)
log_feck(
`squad "${squad_id}" is missing in Flightfile. Example:\n ${chalk.white(
JSON.stringify(
{
[squad_id]: exp_squad
},
undefined,
4
)
)}`
)
} else {
keys(exp_squad).map(unit_id => {
const exp_item = exp_squad[unit_id]
const est_item = est_squad[unit_id]
const expected_unit_spec = exp_squad[unit_id]
const established_unit_conf = est_squad[unit_id]
if (!est_item || !exp_item) {
console.warn(
chalk.red.bold(
`service-unit "${unit_id}" is missing in Flightfile. Example:\n ${chalk.white(
JSON.stringify(
{
[squad_id]: {
[unit_id]: exp_squad[unit_id]
}
},
undefined,
4
)
)}`
)
if (!established_unit_conf || !expected_unit_spec) {
log_feck(
`service-unit "${unit_id}" is missing in Flightfile. Example:\n ${chalk.white(
JSON.stringify(
{
[squad_id]: {
[unit_id]: exp_squad[unit_id]
}
},
undefined,
4
)
)}`
)
} else {
keys(exp_item).map(prop_str => {
const prop = prop_str as keyof FlightCraft
if (exp_item[prop] !== est_item[prop]) {
console.warn(
chalk.red.bold(
`property "${prop}" of entity "${unit_id}" mismatch expectation (expected: "${
exp_item[prop]
}", found: "${est_item[prop]}")`
)
keys(expected_unit_spec).map(prop_str => {
const prop = prop_str as keyof FlightUnitSpec
if (
(expected_unit_spec as Any)[prop] !==
(established_unit_conf as Any)[prop]
) {
log_feck(
`property "${prop}" of entity "${unit_id}" mismatch expectation (expected: "${
(expected_unit_spec as Any)[prop]
}", found: "${established_unit_conf[prop]}")`
)

@@ -85,0 +101,0 @@ }

102

src/lib-utils/fleetctl-utils/flightfile-types.ts

@@ -12,28 +12,25 @@ import {

export type FleetTierConf = Partial<
FleetTierAlt<IdMap<IdMap<Partial<FlightCraft>>>>
>
export type FleetfileBaseConf = FleetOperativeEntry_Mixin & FleetTierConf
export type FleetOpsOperativeConf = Partial<FleetOpsOperativeSpecBase>
export type FleetOpsOperativeConf = Partial<FleetOpsOperativeSpec_Core>
type FleetTierKind = "dev" | "test" | "stage" | "prod" | "replicator"
export type FleetOpsCentral = FleetOperativeEntry_Mixin & {
tiers: IdMap<FleetTierSpec>
}
type FleetTierContext = "ops_only" | "build_only" | "all"
export type FleetOpsBase = {
export type FleetOperativeEntry_Mixin = {
operative: FleetOpsOperativeSpec
tiers: IdMap<FleetTier>
}
export type FleetOpsOperativeSpec = FleetOpsOperativeSpecBase &
FleetOpsOperativeSpecInternal
export type FleetOpsOperativeSpec = FleetOpsOperativeSpec_Core &
FleetOpsOperativeSpec_Intl_
export type FleetOpsOperativeSpecBase = {
export type FleetOpsOperativeSpec_Core = {
fleet_name: Str
dir: Str
tier_id: Str
repos: RepoTypes
repos: ReposCollections_Mixin
}
type FleetOpsOperativeSpecInternal = {
type FleetOpsOperativeSpec_Intl_ = {
_snapshot_details?: {

@@ -46,10 +43,10 @@ tier_id: Str

export type FleetTier = FleetTierAlt<IdMap<IdMap<FlightCraft>>>
export type FleetTierConf = Partial<FleetTier_Base<FlightUnitConf, SshCfg>>
type RepoTypes = {
tiers: IdMap<RepoConf>
sources: IdMap<RepoConf>
}
export type FleetTierSpec = FleetTier_Base<FlightUnitSpec, SshSpec> &
FleetTier_Intl_
type FleetTierAlt<ST> = FleetTierInternal & {
export type FleetTier__Mythical_To_Separate = FleetTierConf
type FleetTier_Base<SquadT, SshT> = {
fleet_name: Str

@@ -59,4 +56,4 @@

tier_id: Str
tier_role: FleetTierKind
tier_context: FleetTierContext
tier_role: FleetTierRole_Tag
tier_context: FleetTierContext_Tag
paths_deprecated_?: {

@@ -71,3 +68,3 @@ log_root: Str

ssh_access?: SshCfg
ssh_access?: SshT

@@ -77,8 +74,19 @@ backup_conf: BackupConf

repos?: RepoTypes
repos?: ReposCollections_Mixin
services: ST
services: IdMap<IdMap<SquadT>>
confs: IdMap<JsonHash>
}
type FleetTierRole_Tag = "dev" | "test" | "stage" | "prod" | "replicator"
type FleetTierContext_Tag = "ops_only" | "build_only" | "all"
type FleetTier_Intl_ = { unit_lut_: IdMap<FlightUnitSpec> }
type ReposCollections_Mixin = {
tiers: IdMap<RepoConf>
sources: IdMap<RepoConf>
}
export type SshCfg = {

@@ -88,7 +96,11 @@ host: Str

user: Str
secret_file?: Str
secret_filename?: Str
}
type FleetTierInternal = { unit_lut_: IdMap<FlightCraft> }
export type SshSpec = Required<SshCfg> & Ssh_Intl_
type Ssh_Intl_ = {
secret_path_: Str
}
export type BackupConf = {

@@ -112,3 +124,3 @@ path: Str

type BackupScheduledAction = {
action: BackupActions
action: BackupAction_Tag
tier: Str

@@ -119,3 +131,3 @@ at?: ChronWallUtc

type BackupActions = "snapshot" | "replicate" | "rotate"
type BackupAction_Tag = "snapshot" | "replicate" | "rotate"

@@ -133,3 +145,9 @@ type WeekdayNumber = 1 | 2 | 3 | 4 | 5 | 6 | 7

type FlightCraftInternalFields = {
export type FlightUnitAny = FlightUnitGeneric | FlightUnitDb | FlightUnitFs
export type FlightUnitSpec = FlightUnit_Intl_ & FlightUnitAny
export type FlightUnitConf = Partial<FlightUnitAny>
type FlightUnit_Intl_ = {
group: Str

@@ -140,3 +158,3 @@ name: Str

type FlightCraftBase = FlightCraftInternalFields & {
type FlightUnit_Core = {
deps?: IdMap<Str | JsonHash>

@@ -147,21 +165,21 @@ repo?: Str

// DB-unit
export type FlightCraftDb = FlightCraftBase & {
export type FlightUnitDb = FlightUnit_Core & {
type: "db"
kind: FlightCraftDb_Kind
kind: FlightUnitDb_KindTag
}
// FS-unit
export type FlightCraftFs = FlightCraftBase & {
export type FlightUnitFs = FlightUnit_Core & {
type: "fs"
kind: FlightCraftFs_Kind
kind: FlightUnitFs_Kind
}
export type FlightCraftGeneric = FlightCraftBase & {
type?: FlightCraftType
export type FlightUnitGeneric = FlightUnit_Core & {
type?: FlightUnit_TypeTag
}
// export type FlightCraftType = Exclude<FlightCraft["type"], undefined>
export type FlightCraftType = "process" | "external" | "db" | "fs"
// export type FlightUnitType = Exclude<FlightUnit["type"], undefined>
export type FlightUnit_TypeTag = "process" | "external" | "db" | "fs"
export type FlightCraftDb_Kind =
export type FlightUnitDb_KindTag =
| "pg"

@@ -173,4 +191,2 @@ | "mongo"

export type FlightCraftFs_Kind = "private" | "public"
export type FlightCraft = FlightCraftGeneric | FlightCraftDb | FlightCraftFs
export type FlightUnitFs_Kind = "private" | "public"

379

src/lib-utils/fleetctl-utils/flightfile-utils.ts

@@ -24,3 +24,10 @@ import {

import { sync as flob } from "fast-glob"
import { FleetTier, FlightCraft, FleetOpsBase } from "./flightfile-types"
import {
FleetTier__Mythical_To_Separate,
FlightUnitSpec,
FleetOpsCentral,
FleetTierSpec,
FleetTierConf,
FleetfileBaseConf
} from "./flightfile-types"

@@ -34,5 +41,8 @@ import {

create_hash_matcher,
HashMatchPattern
HashMatchPattern,
try_else,
uniq
} from "../fun-utils/functional-utils"
import { ensure_file_compiled } from "../sh-utils/build-utils"
import { ssh_spec_from_cfg } from "../sh-utils/ssh-utils"

@@ -51,178 +61,239 @@ export function specify_conf<T extends { [key: string]: any }>(conf: T): T {

verbose_inspection: Str | Undef = undefined
): FleetOpsBase {
): FleetOpsCentral {
try {
return load_flightfile__(flightfile_path_, verbose_inspection)
} catch (err1) {
const entrypoint_flightfile_path = Path.normalize(flightfile_path_)
const flightdir = Path.dirname(entrypoint_flightfile_path)
throw die("Couldn't compose Flightfile for path:", flightdir, err1)
}
}
export function load_flightfile__(
flightfile_path_: string,
verbose_inspection: Str | Undef = undefined
): FleetOpsCentral {
const entrypoint_flightfile_path = Path.normalize(flightfile_path_)
const flightdir = Path.dirname(entrypoint_flightfile_path)
verbose_inspection &&
say("Load all required files for full composite Flightfile")
try {
verbose_inspection &&
say("Load all required files for full composite Flightfile")
ensure_all_conf_files_compiled()
ensure_all_conf_files_compiled()
const tier_specs_path = Path.join(flightdir, "tiers.d")
const tier_specs_path = Path.join(flightdir, "tiers.d")
const available_tier_specs = (() => {
try {
return readdir(tier_specs_path)
} catch (e) {
if (false === is_fleetbase_bootstrapped_and_ready()) {
return [] as Li<Str>
} else {
log_err("caught missing tiers.d repos...", process.argv)
throw e
}
const available_tier_specs = (() => {
try {
return readdir(tier_specs_path)
} catch (e) {
if (false === is_fleetbase_bootstrapped_and_ready()) {
return [] as Li<Str>
} else {
log_err("caught missing tiers.d repos...", process.argv)
throw e
}
})()
}
})()
const base_conf = (read_struct_file(
entrypoint_flightfile_path
) as unknown) as FleetTier
const maybe_local_overrides_conf = {
operative: {},
...try_else(
() =>
read_struct_file(
entrypoint_flightfile_path.replace(/(\.\w+)$/, ".local.$1")
),
{}
)
} as FleetfileBaseConf
verbose_inspection && say("Base Flightfile")
verbose_inspection && say(JSON.stringify(base_conf, undefined, 4))
const standard_base_conf = (read_struct_file(
entrypoint_flightfile_path
) as unknown) as FleetfileBaseConf
const operative_tier_id = os.hostname()
const operative_tier_id = os.hostname()
const operative_spec = {
fleet_name: base_conf.fleet_name,
dir: flightdir,
tier_id: operative_tier_id,
const default_operative_spec = {
fleet_name: standard_base_conf.fleet_name!,
dir: flightdir,
// TODO should be OPERATIVE_tier_id here too - the prop
tier_id: operative_tier_id,
repos: base_conf.repos!
}
repos: standard_base_conf.repos! // TODO should ofc be orderly and actually specify this under "operative" prop sub conf
}
const tier_specs = fold(
available_tier_specs,
{},
(tier_lut_acc, tier_repo_slug) => {
const tier_specific_flightfile_path = Path.join(
flightdir,
"tiers.d",
const base_conf = depth_two_merge(
standard_base_conf,
maybe_local_overrides_conf
)
verbose_inspection && say("Base Op Flightfiles")
verbose_inspection && say(JSON.stringify(base_conf, undefined, 4))
const tier_specs = fold(
available_tier_specs,
{},
(tier_lut_acc, tier_repo_slug) => {
const tier_specific_flightfile_path = Path.join(
flightdir,
"tiers.d",
tier_repo_slug,
"Flightsecrets"
)
const loaded = {
...{ confs: {} }, // no need for ops-only flightfiles to specify an empty "confs"...
...((read_struct_file(
tier_specific_flightfile_path
) as unknown) as FleetTierConf)
}
if (!loaded.spec) {
throw Error(
`Fleetsecrets conf from tier-dir "${tier_repo_slug}" is missing "spec"-conf`
)
}
if (loaded.spec.ssh_access) {
loaded.spec.ssh_access = ssh_spec_from_cfg(
tier_repo_slug,
"Flightsecrets"
loaded.spec.ssh_access
)
}
const loaded = {
...{ confs: {} }, // no need for ops-only flightfiles to specify an empty "confs"...
...((read_struct_file(
tier_specific_flightfile_path
) as unknown) as FleetTier)
}
const tier_id = loaded.spec.tier_id
const tier_id = loaded.spec.tier_id
const verlog = verbose_inspection === tier_id ? say : noop
const verlog = verbose_inspection === tier_id ? say : noop
verlog(
`Reads tier-conf "${tier_specific_flightfile_path}" for tier-id "${tier_id}"`
)
verlog(JSON.stringify(loaded, undefined, 4))
verlog(
`Reads tier-conf "${tier_specific_flightfile_path}" for tier-id "${tier_id}"`
)
verlog(JSON.stringify(loaded, undefined, 4))
// NOTE: we allow multiple tier-definition repos, to separate
// for instance backup-replication-access and actual full
// secrets — and some fleetoperatives may have access to both /
// multiple — so we merge according to a simple two-prop-depth
// rule that promotes properties from full-confs over
// ops-only-confs
const tier_conf_pre_merge = (() => {
// TODO — since we massage the conf as we
// consolidate it into spec — shuffling around the massaged
// one as layer_b could cause information-loss?
// First get a pitch of _all_ available moving parts, sort
// them as needed, THEN do the layer-ordered-merge
// NOTE: this is not a common usage-pattern. Fix later.
const existing_part = tier_lut_acc[tier_id] as FleetTierConf
// NOTE: we allow multiple tier-definition repos, to separate
// for instance backup-replication-access and actual full
// secrets — and some fleetoperatives may have access to both /
// multiple — so we merge according to a simple two-prop-depth
// rule that promotes properties from full-confs over
// ops-only-confs
const tier_conf_pre_merge = (() => {
const curr = tier_lut_acc[tier_id] as FleetTier
if (!existing_part) return loaded
if (!curr) return loaded
verlog(`Encountered a merge-situation for tier-id "${tier_id}"`)
verlog(
`Encountered a merge-situation for tier-id "${tier_id}"`
)
const [layer_a, layer_b, verbose_msg] =
loaded.spec.tier_context === "all"
? [
existing_part,
loaded,
`Currently processed takes property-precedence over previously loaded tier-conf for tier-id "${tier_id}"`
]
: [
loaded,
existing_part,
`Previously loaded takes property-precedence over currently processed tier-conf for tier-id "${tier_id}"`
]
verlog(verbose_msg)
const [a, b, vermsg] =
loaded.spec.tier_context === "all"
? [
curr,
loaded,
`Currently processed takes property-precedence over previously loaded tier-conf for tier-id "${tier_id}"`
]
: [
loaded,
curr,
`Previously loaded takes property-precedence over currently processed tier-conf for tier-id "${tier_id}"`
]
verlog(vermsg)
// const merge_ret = { ...layer_a } as any
const merge_ret = { ...a } as any
// keys(layer_b).map(k => {
// merge_ret[k] = {
// ...(merge_ret[k] || {}),
// ...(layer_b as Any)[k]
// }
// })
const merge_ret = depth_two_merge(layer_a, layer_b)
return merge_ret as typeof layer_a & typeof layer_b
})()
keys(b).map(k => {
merge_ret[k] = {
...(merge_ret[k] || {}),
...(b as Any)[k]
}
})
return merge_ret as typeof a & typeof b
})()
const merged_tier_conf = {
...base_conf,
...tier_conf_pre_merge,
...{ unit_lut_: {} as IdMap<FlightUnitSpec> }
}
const merged_tier_conf = {
...base_conf,
...tier_conf_pre_merge
if (!merged_tier_conf.services) {
throw Error(`No collection of service squads is defined at all`)
}
if (!merged_tier_conf.spec) {
throw Error(`No tier-spec is defined for tier "${tier_id}"`)
}
const is_ops_only =
merged_tier_conf.spec.tier_context === "ops_only"
if (!merged_tier_conf.confs) {
if (is_ops_only) {
merged_tier_conf.confs = {}
} else {
throw Error(
`No confs defined for service units in tier "${tier_id}"`
)
}
}
const units_lut = (merged_tier_conf.unit_lut_ = {} as IdMap<
FlightCraft
>)
const units_lut = merged_tier_conf.unit_lut_
pairs(merged_tier_conf.services).map(([squad_id, units]) => {
pairs(units).map(([flightcraft_id, v]) => {
if (units_lut[flightcraft_id]) {
throw Error(
`All service-units must have uniquely identifying names — prefix with squad-name to ensure it ("${flightcraft_id}").`
)
}
pairs(merged_tier_conf.services).map(([squad_id, units]) => {
pairs(units).map(([flightcraft_id, v]) => {
if (units_lut[flightcraft_id]) {
throw Error(
`All service-units must have uniquely identifying names — prefix with squad-name to ensure it ("${flightcraft_id}").`
)
}
const unit_conf =
tier_conf_pre_merge.confs[flightcraft_id]
if (
!unit_conf &&
tier_conf_pre_merge.spec.tier_context !== "ops_only"
) {
throw Error(
`No conf defined for service-unit "${flightcraft_id}", specify an empty conf if none required to establish omission is deliberate.`
)
}
const unit_conf =
merged_tier_conf.confs![flightcraft_id] ||
(is_ops_only && {})
units_lut[flightcraft_id] = units[flightcraft_id] = {
...v,
group: squad_id as any,
name: flightcraft_id as any,
conf: unit_conf
} as FlightCraft
})
if (!unit_conf) {
throw Error(
`No conf defined for service-unit "${flightcraft_id}" for tier "${tier_id}", specify an empty conf if none required to establish omission is deliberate.`
)
}
units_lut[flightcraft_id] = units[flightcraft_id] = {
...v,
group: squad_id as any,
name: flightcraft_id as any,
conf: unit_conf
} as FlightUnitSpec
})
})
return {
...tier_lut_acc,
[tier_id]: merged_tier_conf
}
return {
...tier_lut_acc,
[tier_id]: merged_tier_conf
}
)
const composite_conf: FleetOpsBase = {
operative: operative_spec,
tiers: tier_specs
}
)
if (verbose_inspection) {
const dump_conf = { ...composite_conf.tiers[verbose_inspection] }
delete dump_conf.unit_lut_
delete dump_conf.confs
const composite_conf: FleetOpsCentral = {
operative: default_operative_spec,
tiers: tier_specs
}
say(`finally merged composite_conf`)
say(JSON.stringify(dump_conf, undefined, 4))
}
if (verbose_inspection) {
const dump_conf = { ...composite_conf.tiers[verbose_inspection] }
delete dump_conf.unit_lut_
delete dump_conf.confs
return composite_conf
} catch (err1) {
throw die("Couldn't compose Flightfile for path:", flightdir, err1)
say(`finally merged composite_conf`)
say(JSON.stringify(dump_conf, undefined, 4))
}
return composite_conf
}
let loaded_conf_: FleetOpsBase
let loaded_conf_: FleetOpsCentral
export function flightfile_from_first_arg(): FleetOpsBase {
export function flightfile_from_first_arg(): FleetOpsCentral {
if (loaded_conf_) {

@@ -243,3 +314,6 @@ _D(`flightfile_from_first_arg() -> already loaded, returns`)

export function select_tier(fleetfile: FleetOpsBase, tier_id: Str) {
export function select_tier(
fleetfile: FleetOpsCentral,
tier_id: Str
): FleetTierSpec {
const tier = fleetfile.tiers[tier_id]

@@ -257,5 +331,5 @@ if (!tier) {

export function select_unit(
flight_tier_conf: FleetTier,
flight_tier_conf: FleetTierSpec,
unit_name: Str
): FlightCraft | undefined {
): FlightUnitSpec | undefined {
return flight_tier_conf.unit_lut_[unit_name]

@@ -265,6 +339,6 @@ }

export function select_units(
tier: FleetTier,
hash_pattern: HashMatchPattern<FlightCraft>
): Li<FlightCraft> {
const matcher = create_hash_matcher<FlightCraft>(hash_pattern)
tier: FleetTierSpec,
hash_pattern: HashMatchPattern<FlightUnitSpec>
): Li<FlightUnitSpec> {
const matcher = create_hash_matcher<FlightUnitSpec>(hash_pattern)
return vals(tier.unit_lut_).filter(matcher)

@@ -303,1 +377,18 @@ }

}
function depth_two_merge<T>(layer_a: T, layer_b: T): T {
const mut_ret = {} as any
const all_keys = uniq([...keys(layer_a), ...keys(layer_b)])
// Crude mutating operation here. works. to tired to bother with the clumsy
// es-syntax
all_keys.map(
k =>
(mut_ret[k] = {
...((layer_a as Any)[k] || {}),
...((layer_b as Any)[k] || {})
})
)
return mut_ret as T // typeof layer_a & typeof layer_b
}

47

src/lib-utils/sh-utils/ssh-utils.ts
import {
SshCfg,
FleetOpsBase
FleetOpsCentral,
SshSpec
} from "../../lib-utils/fleetctl-utils/flightfile-types"

@@ -9,3 +10,15 @@ import { Str, Bool } from "../../lib-utils/fun-utils/onyx-std-types"

export function normalize_ssh_cfg(in_cfg: SshAccessCfg): RemoteSshAccess {
export function ssh_spec_from_cfg(
tier_repo_slug: Str,
in_cfg: SshCfg
): SshSpec {
const secret_filename = in_cfg.secret_filename || "ssh-secret"
const secret_path = Path.join(
".",
"tiers.d",
tier_repo_slug,
secret_filename
)
return {

@@ -15,26 +28,22 @@ host: in_cfg.host,

user: in_cfg.user || "",
secret_file: in_cfg.secret_file || "",
remote_path: in_cfg.remote_path || ""
secret_filename: secret_filename,
secret_path_: secret_path
// remote_path: in_cfg.remote_path || ""
}
}
export function render_ssh_url(cfg: RemoteSshAccess): Str {
export function render_ssh_url(cfg: SshSpec): Str {
return (cfg.user ? cfg.user + "@" : "") + cfg.host
}
export function render_ssh_secret_filepath(
fleetfile: FleetOpsBase,
sourced_tier_id: Str
): Str {
const sourced_tier = select_tier(fleetfile, sourced_tier_id)
return Path.join(
fleetfile.operative.dir,
"tiers.d",
sourced_tier_id,
"ssh-secret"
)
}
// export function render_ssh_secret_filepath(
// fleetfile: FleetOpsCentral,
// sourced_tier_id: Str
// ): Str {
// const sourced_tier = select_tier(fleetfile, sourced_tier_id)
// return sourced_tier.spec.ssh_access!.secret_file!
// }
export type SshAccessCfg = SshCfg & { remote_path?: Str }
// export type SshAccessSpec = SshSpec & { remote_path?: Str }
export type RemoteSshAccess = Required<SshAccessCfg>
// export type RemoteSshAccess = Required<SshAccessSpec>

23

src/tooling/fctl-ops/repo-ctl.ts

@@ -74,2 +74,14 @@ /*

function ensure_flightcontrol_freshness() {
// TODO should be done async, aight!
//
// REF OUTPUT:
// Package Current Wanted Latest Location
// @flight-control/devops-utils 0.8.15 0.8.16 0.8.16 fleetbase-wacebe
//
// if (sh(`npm outdated`).split(/\n/).any(v => v.match(/^@flight-control\/devops-utils/))) {
// sh(`npm install @flight-control/devops-utils`)
// }
}
function ensure_all_repos() {

@@ -175,9 +187,6 @@ const ops_root = fleetfile.operative.dir

[
`git add . && git commit -m 'fleetctl commit from tier-env "${
// `git status --short | head -c1 | wc -c && if [[ $(git status --short | head -c1 | wc -c) -ne 0 ]]; then ( git add . && git commit -m 'fleetctl commit from tier-env "${
`git status --short | grep . && (( git add . && git commit -m 'fleetctl commit from tier-env "${
fleetfile.operative.tier_id
}"' && git push`
// `echo 'fctl-commit from tier-env "${get_operating_user()} ${
// fleetfile.operative.tier_id
// }"'`
//
}"' && git push ) || exit 0) || ( echo "no changes" )`
]

@@ -221,3 +230,3 @@ )

const path = root_path_parts.join("/")
return sh(`cd ${path} && ${cmd_parts.join(" ")}`, { do_throw: true })
return sh(`cd ${path} && (${cmd_parts.join(" ")})`, { do_throw: true })
}

@@ -224,0 +233,0 @@

10

src/tooling/fleet-data-ctl/backup-file-level-utils.ts

@@ -31,4 +31,4 @@ import {

import {
FlightCraft,
FleetTier
FlightUnitSpec,
FleetTierSpec
} from "../../lib-utils/fleetctl-utils/flightfile-types"

@@ -286,3 +286,3 @@ import { try_else } from "../../lib-utils/fun-utils/functional-utils"

snapshot_dir: Str,
unit: FlightCraft,
unit: FlightUnitSpec,
additional_subdir_part?: Str

@@ -327,3 +327,5 @@ ): Str {

export function ensure_correct_backups_dir_owner(tierconf: FleetTier): Void {
export function ensure_correct_backups_dir_owner(
tierconf: FleetTierSpec
): Void {
const path = tierconf.spec.backup_conf.path

@@ -330,0 +332,0 @@ const user = tierconf.spec.backup_conf.ensure_user

47

src/tooling/fleet-data-ctl/backup-replicate.ts

@@ -31,5 +31,6 @@ /*

import {
FleetTier,
FleetOpsBase,
SshCfg
FleetTierSpec,
FleetOpsCentral,
SshCfg,
SshSpec
} from "../../lib-utils/fleetctl-utils/flightfile-types"

@@ -39,12 +40,15 @@ import { Str, Bool } from "../../lib-utils/fun-utils/onyx-std-types"

import {
RemoteSshAccess,
render_ssh_url,
normalize_ssh_cfg,
render_ssh_secret_filepath
ssh_spec_from_cfg
} from "../../lib-utils/sh-utils/ssh-utils"
import os from "os"
export function test_remote_ssh_access(cfg: RemoteSshAccess): Bool {
export function test_remote_ssh_access(cfg: SshSpec): Bool {
const ssh_url = render_ssh_url(cfg)
const remote_cmd = "pwd"
if (!existsf(cfg.secret_path_)) {
throw Error(`can't find ssh secrets file "${cfg.secret_path_}"`)
}
return (

@@ -55,3 +59,3 @@ try_else(

normalize_paths_within_cmd_string(
`sshpass -p $(cat "${cfg.secret_file}") ` +
`sshpass -p $(cat "${cfg.secret_path_}") ` +
`ssh -p ${cfg.port} ${ssh_url} ${remote_cmd}`

@@ -79,4 +83,4 @@ ),

export function replicate_remote_backups(
fleetfile: FleetOpsBase,
operative_tier: FleetTier,
fleetfile: FleetOpsCentral,
operative_tier: FleetTierSpec,
sourced_tier_id: Str

@@ -86,11 +90,12 @@ ) {

const ssh_cfg = normalize_ssh_cfg({
secret_file: render_ssh_secret_filepath(fleetfile, sourced_tier_id),
...sourced_tier.spec.ssh_access!
})
// const ssh_cfg = normalize_ssh_cfg({
// // secret_path_: render_ssh_secret_filepath(fleetfile, sourced_tier_id),
// ...sourced_tier.spec.ssh_access!
// })
const ssh_spec = sourced_tier.spec.ssh_access!
if (!test_remote_ssh_access(ssh_cfg)) {
if (!test_remote_ssh_access(ssh_spec)) {
return die(
1,
`ssh-access failure for ${ssh_cfg.host}, running as user ${
`ssh-access failure for ${ssh_spec.host}, running as user ${
os.userInfo({ encoding: "utf8" }).username

@@ -101,3 +106,3 @@ }`

const remote_ssh_url = render_ssh_url(ssh_cfg)
const remote_ssh_url = render_ssh_url(ssh_spec)

@@ -115,5 +120,5 @@ const remote_dailies_path = normalize_paths_within_cmd_string(

const dailies = sh(
`sshpass -p $(cat "${ssh_cfg.secret_file}") ` +
`sshpass -p $(cat "${ssh_spec.secret_path_}") ` +
`ssh -p ${
ssh_cfg.port
ssh_spec.port
} ${remote_ssh_url} ls -1 ${remote_dailies_path}`,

@@ -160,5 +165,5 @@ { silent: true }

const replicate_snapshot_cmd = normalize_paths_within_cmd_string(
`sshpass -p $(cat "${ssh_cfg.secret_file}") ` +
`sshpass -p $(cat "${ssh_spec.secret_path_}") ` +
`rsync -az ${is_dry_run() ? "-n" : ""} -e 'ssh -p ${
ssh_cfg.port
ssh_spec.port
}' ${remote_ssh_url}:${remote_dailies_path}/${snapshot_fname}/* ${temp_snapshot_target_path}/`

@@ -165,0 +170,0 @@ )

4

src/tooling/fleet-data-ctl/backup-rotate-periodizations.ts

@@ -1,2 +0,2 @@

import { FleetOpsBase } from "../../lib-utils/fleetctl-utils/flightfile-types"
import { FleetOpsCentral } from "../../lib-utils/fleetctl-utils/flightfile-types"
import {

@@ -13,3 +13,3 @@ read_cullable_snapshots,

export function rotate_snapshot_periodizations(
fleetfile: FleetOpsBase,
fleetfile: FleetOpsCentral,
snapshot_tier_id: Str

@@ -16,0 +16,0 @@ ) {

45

src/tooling/fleet-data-ctl/backup-snapshot-restore.ts

@@ -29,5 +29,5 @@ // NOTE: OPT:

import {
FleetTier,
FlightCraft,
FleetOpsBase
FleetTierSpec,
FlightUnitSpec,
FleetOpsCentral
} from "../../lib-utils/fleetctl-utils/flightfile-types"

@@ -63,3 +63,6 @@ import {

export function backup_snapshot_now(fleetfile: FleetOpsBase, tier: FleetTier) {
export function backup_snapshot_now(
fleetfile: FleetOpsCentral,
tier: FleetTierSpec
) {
// TODO: ugly - but helpful atm, since dump is run as sudo, without

@@ -74,3 +77,3 @@ // finegrained user<->group ties...

snapshot_dir: Str,
target_tier: FleetTier,
target_tier: FleetTierSpec,
unit_filter: Rex,

@@ -100,3 +103,3 @@ restore_mode: "dev" | "prod",

const full_fleet = {
snapshot(fleetfile: FleetOpsBase, operative_tier: FleetTier) {
snapshot(fleetfile: FleetOpsCentral, operative_tier: FleetTierSpec) {
const {

@@ -127,3 +130,3 @@ final: snapshot_dir,

snapshot_dir: Str,
target_tier: FleetTier,
target_tier: FleetTierSpec,
unit_filter: Rex,

@@ -183,3 +186,3 @@ restore_mode: "dev" | "prod",

const fleetfile_snapper = {
snapshot(fleetfile: FleetOpsBase, snapshot_dir: Str) {
snapshot(fleetfile: FleetOpsCentral, snapshot_dir: Str) {
say("\nDo Fleetcomposite snapshot.\n")

@@ -205,3 +208,3 @@

readf(`${snapshot_dir}/Fleetsnap`)
) as FleetOpsBase
) as FleetOpsCentral

@@ -213,3 +216,3 @@ return fleetsnap

const pgdb = {
snapshot(snapshot_dir: Str, tier: FleetTier) {
snapshot(snapshot_dir: Str, tier: FleetTierSpec) {
say("\nDo PGDB SQL-dump snapshots!\n")

@@ -219,3 +222,3 @@

function snapshot_pgdb_unit(unit: FlightCraft) {
function snapshot_pgdb_unit(unit: FlightUnitSpec) {
const sub_dir = make_snapshot_unit_sub_dir(snapshot_dir, unit)

@@ -232,4 +235,4 @@ const snap_dump_filename = `${sub_dir}/pg_dump.sql`

snapshot_dir: Str,
target_tier: FleetTier,
persisted_tier: FleetTier,
target_tier: FleetTierSpec,
persisted_tier: FleetTierSpec,
chosen_units: Li<Str>,

@@ -270,3 +273,3 @@ obfuscate_privacy_intell: Bool,

function restore_pgdb_unit(unit: FlightCraft) {
function restore_pgdb_unit(unit: FlightUnitSpec) {
const subdir = make_snapshot_unit_sub_dir(snapshot_dir, unit)

@@ -302,3 +305,3 @@ const full_dump_filename = `${subdir}/pg_dump.sql`

function maybe_obfuscate_sql_pii_data(
tier: FleetTier,
tier: FleetTierSpec,
source_sql_dump_path: Str,

@@ -341,3 +344,3 @@ obfuscate_personal_identifiable_information: Bool

const fs_vols = {
snapshot(snapshot_dir: Str, operative_tier: FleetTier) {
snapshot(snapshot_dir: Str, operative_tier: FleetTierSpec) {
say("\nFS backup\n")

@@ -347,3 +350,3 @@

function snapshot_fs_vol(unit: FlightCraft) {
function snapshot_fs_vol(unit: FlightUnitSpec) {
const fs_vol_path = Path.join(unit.conf.path as string, "*")

@@ -361,4 +364,4 @@ const sub_dir = `${make_snapshot_unit_sub_dir(

snapshot_dir: Str,
target_tier: FleetTier,
persisted_tier: FleetTier,
target_tier: FleetTierSpec,
persisted_tier: FleetTierSpec,
chosen_units: Li<Str>,

@@ -373,3 +376,3 @@ skip_missing: Bool

function restore_fs_vol(unit: FlightCraft) {
function restore_fs_vol(unit: FlightUnitSpec) {
const sub_dir = `${make_snapshot_unit_sub_dir(

@@ -401,3 +404,3 @@ snapshot_dir,

const snapshot_stats = {
snapshot(fleetfile: FleetOpsBase, snapshot_dir: Str) {
snapshot(fleetfile: FleetOpsCentral, snapshot_dir: Str) {
const units_path = Path.join(snapshot_dir, "units")

@@ -404,0 +407,0 @@ const stats = readdir(units_path)

6

src/tooling/staging-static-analysis/analysis-utils/fleet-asg-types.ts

@@ -12,3 +12,3 @@ import {

squads: {} as IdMap<Squad>,
flightcrafts: {} as IdMap<FlightCraft> // TODO - should be 'entities' or so instead
flightcrafts: {} as IdMap<FlightUnit> // TODO - should be 'entities' or so instead
// msgio_decls: create_msgio_decls() as MessagingDecls

@@ -51,3 +51,3 @@ }

readme_doc: Str
flightcrafts: IdMap<FlightCraft>
flightcrafts: IdMap<FlightUnit>
databases: IdMap<DbEntry>

@@ -61,3 +61,3 @@ }

export type FlightCraft = {
export type FlightUnit = {
filepath: Str

@@ -64,0 +64,0 @@ name: Str

8

src/tooling/staging-static-analysis/analysis-utils/fleet-doc-asg.ts

@@ -21,3 +21,3 @@ const _D = console.log

Squad,
FlightCraft,
FlightUnit,
MsgIoApiDecl,

@@ -43,3 +43,3 @@ MsgIoSubDecl,

export function ensure_flightcraft(filename: Str): FlightCraft {
export function ensure_flightcraft(filename: Str): FlightUnit {
const squad_key = derive_squad_key_from_filename(filename)

@@ -50,3 +50,3 @@ const flightcraft_key = derive_flightcraft_key_from_filename(filename)

let flightcraft = squad.flightcrafts[flightcraft_key] as
| FlightCraft
| FlightUnit
| undefined

@@ -87,3 +87,3 @@

export function ensure_flightcraft_by_filename(filename: Str): FlightCraft {
export function ensure_flightcraft_by_filename(filename: Str): FlightUnit {
return ensure_flightcraft(filename)

@@ -90,0 +90,0 @@ }

2

src/tooling/staging-static-analysis/analysis-utils/scrape-ts-ast-for-api-and-msgio.ts

@@ -76,3 +76,3 @@ import * as ts from "typescript"

const matter_stack_ = [flightcraft__] as Li<
Bat.MsgDeclsHolder | Bat.FlightCraft
Bat.MsgDeclsHolder | Bat.FlightUnit
>

@@ -79,0 +79,0 @@

2

src/tooling/staging-static-analysis/generate-docs.ts
import * as ts from "typescript"
import chalk from "chalk"
import { flightfile_from_first_arg } from "../../lib-utils/fleetctl-utils/flightfile-utils"
import { FlightCraft } from "../../lib-utils/fleetctl-utils/flightfile-types"
import { FlightUnitSpec } from "../../lib-utils/fleetctl-utils/flightfile-types"
import { uniq } from "../../lib-utils/fun-utils/functional-utils"

@@ -6,0 +6,0 @@

14

src/tooling/staging-static-analysis/generate-graph-asg.ts

@@ -11,4 +11,4 @@ import * as Grhl from "../../lib-utils/graphul/ghrl-main"

import {
FlightCraft,
FleetTier
FlightUnitSpec,
FleetTierSpec
} from "../../lib-utils/fleetctl-utils/flightfile-types"

@@ -22,3 +22,3 @@ import { _D } from "../../lib-utils/sh-utils/cli-utils"

export function generate_graph(
flight_conf: FleetTier,
flight_conf: FleetTierSpec,
wanted_graph_type_: GraphType,

@@ -331,3 +331,3 @@ DEBUG_PSEUDO_ELEMENTS: Bool

function define_flightfile_service_elements(service: FlightCraft) {
function define_flightfile_service_elements(service: FlightUnitSpec) {
// _D("defines ", service.name)

@@ -362,3 +362,3 @@ const service_label = service.name

function add_db_deps(service: FlightCraft) {
function add_db_deps(service: FlightUnitSpec) {
pairs(service.deps || {}).map(([local_dep_id_, dep_cfg_]) => {

@@ -390,3 +390,3 @@ const dep_id_ = dep_cfg_

function define_service_bind_plane_edges(service: FlightCraft) {
function define_service_bind_plane_edges(service: FlightUnitSpec) {
if (service.type !== "db") {

@@ -413,3 +413,3 @@ edge(service.name, "vernemq", pE.mqtt_wss, -1)

function define_service_logical_plane_edges__TEMP_RND_DATA(
service: FlightCraft
service: FlightUnitSpec
) {

@@ -416,0 +416,0 @@ const foo_bar_deps = keys(flight_conf.services) // .map(v => v.name)

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc