Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

@clickup/ci-storage-cdk

Package Overview
Dependencies
Maintainers
13
Versions
6
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@clickup/ci-storage-cdk - npm Package Compare versions

Comparing version 2.10.291 to 2.10.292

dist/internal/cloudConfigBuild.d.ts
dist/internal/cloudConfigBuild.d.ts.map
dist/internal/cloudConfigBuild.js
dist/internal/cloudConfigBuild.js.map
dist/internal/cloudConfigYamlDump.d.ts
dist/internal/cloudConfigYamlDump.d.ts.map
dist/internal/cloudConfigYamlDump.js
dist/internal/cloudConfigYamlDump.js.map
dist/internal/dedent.d.ts
dist/internal/dedent.d.ts.map
dist/internal/dedent.js
dist/internal/dedent.js.map
src/internal/cloudConfigBuild.ts
src/internal/cloudConfigYamlDump.ts
src/internal/dedent.ts

18

dist/CiStorage.d.ts

@@ -5,3 +5,3 @@ import { Duration } from "aws-cdk-lib";

import type { IKeyPair, ISecurityGroup, IVpc } from "aws-cdk-lib/aws-ec2";
import { LaunchTemplate, Instance } from "aws-cdk-lib/aws-ec2";
import { LaunchTemplate, Instance, CfnVolume } from "aws-cdk-lib/aws-ec2";
import type { RoleProps } from "aws-cdk-lib/aws-iam";

@@ -81,3 +81,7 @@ import { Role } from "aws-cdk-lib/aws-iam";

imageSsmName: string;
/** Size of the root volume. */
/** IOPS of the docker volume. */
volumeIops: number;
/** Throughput of the docker volume in MiB/s. */
volumeThroughput: number;
/** Size of the docker volume. */
volumeGb: number;

@@ -120,3 +124,6 @@ /** Full name of the Instance type. */

readonly keyPairPrivateKeySecretName: string;
readonly role: Role;
readonly roles: {
runner: Role;
host: Role;
};
readonly launchTemplate: LaunchTemplate;

@@ -126,8 +133,5 @@ readonly autoScalingGroup: AutoScalingGroup;

readonly hostInstances: Instance[];
readonly hostVolumes: CfnVolume[];
constructor(scope: Construct, key: string, props: CiStorageProps);
}
/**
* Removes leading indentation from all lines of the text.
*/
export declare function dedent(text: string): string;
//# sourceMappingURL=CiStorage.d.ts.map

333

dist/CiStorage.js

@@ -6,3 +6,3 @@ "use strict";

Object.defineProperty(exports, "__esModule", { value: true });
exports.dedent = exports.CiStorage = void 0;
exports.CiStorage = void 0;
const aws_cdk_lib_1 = require("aws-cdk-lib");

@@ -16,146 +16,8 @@ const aws_autoscaling_1 = require("aws-cdk-lib/aws-autoscaling");

const constructs_1 = require("constructs");
const js_yaml_1 = __importDefault(require("js-yaml"));
const padStart_1 = __importDefault(require("lodash/padStart"));
const range_1 = __importDefault(require("lodash/range"));
const cloudConfigBuild_1 = require("./internal/cloudConfigBuild");
const cloudConfigYamlDump_1 = require("./internal/cloudConfigYamlDump");
const namer_1 = require("./internal/namer");
/**
* Builds a reusable and never changing cloud config to be passed to the
* instance's CloudInit service.
*/
function buildCloudConfig({ fqdn, ghTokenSecretName, ghDockerComposeDirectoryUrl, keyPairPrivateKeySecretName, }) {
if (!ghDockerComposeDirectoryUrl.match(/^([^#]+)(?:#([^:]*):(.*))?$/s)) {
throw ("Cannot parse ghDockerComposeDirectoryUrl. It should be in format: " +
"https://github.com/owner/repo[#[branch]:/directory/with/compose/]");
}
const repoUrl = RegExp.$1;
const branch = RegExp.$2 || "";
const path = (RegExp.$3 || ".").replace(/^\/+|\/+$/gs, "");
return {
fqdn: fqdn || undefined,
apt_sources: [
{
source: "deb https://cli.github.com/packages stable main",
keyid: "23F3D4EA75716059",
filename: "github-cli.list",
},
{
source: "deb https://download.docker.com/linux/ubuntu $RELEASE stable",
keyid: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88",
filename: "docker.list",
},
],
packages: [
"awscli",
"gh",
"docker-ce",
"docker-ce-cli",
"containerd.io",
"docker-compose-plugin",
"git",
"gosu",
"mc",
"curl",
"apt-transport-https",
"ca-certificates",
],
write_files: [
{
path: "/etc/sysctl.d/enable-ipv4-forwarding.conf",
content: dedent(`
net.ipv4.conf.all.forwarding=1
`),
},
{
path: "/var/lib/cloud/scripts/per-once/increase-docker-shutdown-timeout.sh",
permissions: "0755",
content: dedent(`
#!/bin/bash
sed -i -E '/TimeoutStartSec=.*/a TimeoutStopSec=3600' /usr/lib/systemd/system/docker.service
systemctl daemon-reload
`),
},
{
path: "/var/lib/cloud/scripts/per-once/switch-ssm-user-to-ubuntu-on-login.sh",
permissions: "0755",
content: dedent(`
#!/bin/bash
sed -i -E '/ExecStart=/i Environment="ENV=/etc/profile.ssm-user"' /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service
echo '[ "$0$@" = "sh" ] && ENV= sudo -u ubuntu -i' > /etc/profile.ssm-user
systemctl daemon-reload
systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true
`),
},
{
path: "/var/lib/cloud/scripts/per-boot/run-docker-compose-on-boot.sh",
permissions: "0755",
content: dedent(`
#!/bin/bash
echo "*/1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh 2>&1 | logger -t run-docker-compose" > /etc/cron.d/run-docker-compose
exec /home/ubuntu/run-docker-compose.sh
`),
},
{
path: "/home/ubuntu/run-docker-compose.sh",
owner: "ubuntu:ubuntu",
permissions: "0755",
defer: true,
content: dedent(`
#!/bin/bash
set -e -o pipefail
# Switch to non-privileged user if running as root.
if [[ $(whoami) != "ubuntu" ]]; then
exec gosu ubuntu:ubuntu "$BASH_SOURCE"
fi
# Ensure there is only one instance of this script running.
exec {FD}<$BASH_SOURCE
flock -n $FD || { echo "Already running."; exit 0; }
# Load private and public keys from Secrets Manager to ~/.ssh.
region=$(ec2metadata --availability-zone | sed "s/[a-z]$//")
mkdir -p ~/.ssh && chmod 700 ~/.ssh
aws secretsmanager get-secret-value --region "$region" \\
--secret-id "${keyPairPrivateKeySecretName}" \\
--query SecretString --output text \\
> ~/.ssh/ci-storage
chmod 600 ~/.ssh/ci-storage
ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub
# Load GitHub PAT from Secrets Manager and login to GitHub.
aws secretsmanager get-secret-value --region "$region" \\
--secret-id "${ghTokenSecretName}" \\
--query SecretString --output text \\
| gh auth login --with-token
gh auth setup-git
# Pull the repository and run docker compose.
mkdir -p ~/git && cd ~/git
if [[ ! -d .git ]]; then
git clone -n --depth=1 --filter=tree:0 ${branch ? `-b "${branch}"` : ""} "${repoUrl}" .
if [[ "${path}" != "." ]]; then
git sparse-checkout set --no-cone "${path}"
fi
git checkout
else
git pull --rebase
fi
sudo usermod -aG docker ubuntu
GH_TOKEN=$(gh auth token) exec sg docker -c 'cd "${path}" && docker compose pull && exec docker compose up --build -d'
`),
},
{
path: "/home/ubuntu/.bash_profile",
owner: "ubuntu:ubuntu",
permissions: "0644",
defer: true,
content: dedent(`
#!/bin/bash
if [ -d ~/git/"${path}" ]; then
cd ~/git/"${path}"
echo '$ docker compose ps'
docker --log-level=ERROR compose ps --format "table {{.Service}}\\t{{.Status}}\\t{{.Ports}}"
echo
fi
`),
},
],
};
}
/**
* A reusable Construct to launch ci-storage infra in some other stack. This

@@ -189,2 +51,3 @@ * class is meant to be put in a public domain and then used in any project.

this.hostInstances = [];
this.hostVolumes = [];
const keyNamer = (0, namer_1.namer)(key);

@@ -198,52 +61,50 @@ this.vpc = props.vpc;

});
this.keyPair = aws_ec2_1.KeyPair.fromKeyPairName(this, "KeyPair", keyPair.keyPairName);
this.keyPair = aws_ec2_1.KeyPair.fromKeyPairName(this, (0, namer_1.namer)("key", "pair").pascal, keyPair.keyPairName);
this.keyPairPrivateKeySecretName = `ec2-ssh-key/${this.keyPair.keyPairName}/private`;
}
{
const id = (0, namer_1.namer)("role");
this.role = new aws_iam_1.Role(this, id.pascal, {
roleName: (0, namer_1.namer)("instance", "profile", "role").pathPascalFrom(this),
assumedBy: new aws_iam_1.ServicePrincipal("ec2.amazonaws.com"),
managedPolicies: [
aws_iam_1.ManagedPolicy.fromAwsManagedPolicyName("service-role/AmazonEC2RoleforSSM"),
aws_iam_1.ManagedPolicy.fromAwsManagedPolicyName("CloudWatchAgentServerPolicy"),
],
inlinePolicies: {
...props.inlinePolicies,
CiStorageKeyPairPolicy: aws_iam_1.PolicyDocument.fromJson({
Version: "2012-10-17",
Statement: [
{
Effect: "Allow",
Action: ["secretsmanager:GetSecretValue"],
Resource: [
aws_cdk_lib_1.Stack.of(this).formatArn({
service: "secretsmanager",
resource: "secret",
resourceName: `${this.keyPairPrivateKeySecretName}*`,
arnFormat: aws_cdk_lib_1.ArnFormat.COLON_RESOURCE_NAME,
}),
],
},
],
}),
CiStorageGhTokenPolicy: aws_iam_1.PolicyDocument.fromJson({
Version: "2012-10-17",
Statement: [
{
Effect: "Allow",
Action: ["secretsmanager:GetSecretValue"],
Resource: [
aws_cdk_lib_1.Stack.of(this).formatArn({
service: "secretsmanager",
resource: "secret",
resourceName: `${props.ghTokenSecretName}*`,
arnFormat: aws_cdk_lib_1.ArnFormat.COLON_RESOURCE_NAME,
}),
],
},
],
}),
},
});
this.roles = Object.fromEntries(["runner", "host"].map((kind) => [
kind,
new aws_iam_1.Role(this, (0, namer_1.namer)(kind, "role").pascal, {
roleName: (0, namer_1.namer)(kind, "role").pathPascalFrom(this),
assumedBy: new aws_iam_1.ServicePrincipal("ec2.amazonaws.com"),
managedPolicies: [
aws_iam_1.ManagedPolicy.fromAwsManagedPolicyName("service-role/AmazonEC2RoleforSSM"),
aws_iam_1.ManagedPolicy.fromAwsManagedPolicyName("CloudWatchAgentServerPolicy"),
],
inlinePolicies: {
...props.inlinePolicies,
[(0, namer_1.namer)(keyNamer, "key", "pair", "policy").pascal]: new aws_iam_1.PolicyDocument({
statements: [
new aws_iam_1.PolicyStatement({
actions: ["secretsmanager:GetSecretValue"],
resources: [
aws_cdk_lib_1.Stack.of(this).formatArn({
service: "secretsmanager",
resource: "secret",
resourceName: `${this.keyPairPrivateKeySecretName}*`,
arnFormat: aws_cdk_lib_1.ArnFormat.COLON_RESOURCE_NAME,
}),
],
}),
],
}),
[(0, namer_1.namer)(keyNamer, "gh", "token", "policy").pascal]: new aws_iam_1.PolicyDocument({
statements: [
new aws_iam_1.PolicyStatement({
actions: ["secretsmanager:GetSecretValue"],
resources: [
aws_cdk_lib_1.Stack.of(this).formatArn({
service: "secretsmanager",
resource: "secret",
resourceName: `${props.ghTokenSecretName}*`,
arnFormat: aws_cdk_lib_1.ArnFormat.COLON_RESOURCE_NAME,
}),
],
}),
],
}),
},
}),
]));
}

@@ -255,3 +116,3 @@ {

{
const userData = aws_ec2_1.UserData.custom(yarnDumpCloudConfig(buildCloudConfig({
const userData = aws_ec2_1.UserData.custom((0, cloudConfigYamlDump_1.cloudConfigYamlDump)((0, cloudConfigBuild_1.cloudConfigBuild)({
fqdn: "",

@@ -269,3 +130,3 @@ ghTokenSecretName: props.ghTokenSecretName,

keyPair: this.keyPair,
role: this.role, // LaunchTemplate creates InstanceProfile internally
role: this.roles.runner, // LaunchTemplate creates InstanceProfile internally
blockDevices: [

@@ -348,3 +209,25 @@ {

: "";
const userData = aws_ec2_1.UserData.custom(yarnDumpCloudConfig(buildCloudConfig({
// Unfortunately, there is no way in CDK to auto re-attach the volume to
// an instance if that instance gets replaced. This is because
// CloudFormation first launches a new instance while keeping the old
// instance still running, so the volume can't be attached to the new
// instance - it's already attached to the old one. The solution we use
// here is to do the volume attachment via cloud-config at the new
// instance's initial boot: it first stops the old instance from the new
// one ("aws ec2 stop-instances"), then detaches the volume, and then
// attaches it to the current instance. See logic in
// cloudConfigBuild.ts.
const volumeId = (0, namer_1.namer)(id, "volume");
const volume = new aws_ec2_1.CfnVolume(this, volumeId.pascal, {
availabilityZone: this.vpc.availabilityZones[0],
autoEnableIo: true,
encrypted: true,
iops: props.host.volumeIops,
throughput: props.host.volumeThroughput,
size: props.host.volumeGb,
volumeType: "gp3",
});
aws_cdk_lib_1.Tags.of(volume).add("Name", volumeId.pathKebabFrom(this));
this.hostVolumes.push(volume);
const userData = aws_ec2_1.UserData.custom((0, cloudConfigYamlDump_1.cloudConfigYamlDump)((0, cloudConfigBuild_1.cloudConfigBuild)({
fqdn,

@@ -354,2 +237,3 @@ ghTokenSecretName: props.ghTokenSecretName,

keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName,
mount: { volumeId: volume.attrVolumeId, path: "/mnt" },
})));

@@ -359,5 +243,6 @@ const instance = new aws_ec2_1.Instance(this, (0, namer_1.namer)(id, (0, namer_1.namer)("instance")).pascal, {

securityGroup: this.securityGroup,
availabilityZone: this.vpc.availabilityZones[0],
instanceType: new aws_ec2_1.InstanceType(props.host.instanceType),
machineImage,
role: this.role,
role: this.roles.host,
keyPair: this.keyPair,

@@ -368,3 +253,3 @@ userData,

deviceName: "/dev/sda1",
volume: aws_ec2_1.BlockDeviceVolume.ebs(props.host.volumeGb, {
volume: aws_ec2_1.BlockDeviceVolume.ebs(20, {
encrypted: true,

@@ -390,2 +275,43 @@ volumeType: aws_ec2_1.EbsDeviceVolumeType.GP2,

}
{
const id = (0, namer_1.namer)("host", "volume", "policy");
const conditions = {
StringEquals: {
["ec2:ResourceTag/aws:cloudformation:stack-name"]: aws_cdk_lib_1.Stack.of(this).stackName,
},
};
this.roles.host.attachInlinePolicy(new aws_iam_1.Policy(this, id.pascal, {
policyName: (0, namer_1.namer)(keyNamer, id).pascal,
statements: [
new aws_iam_1.PolicyStatement({
actions: ["ec2:DescribeVolumes", "ec2:DescribeInstances"],
resources: ["*"],
// Describe* don't support resource-level permissions and
// conditions.
}),
new aws_iam_1.PolicyStatement({
actions: [
"ec2:StopInstances",
"ec2:DetachVolume",
"ec2:AttachVolume",
],
conditions, // filter by conditions, not by resource ARNs
resources: [
aws_cdk_lib_1.Stack.of(this).formatArn({
service: "ec2",
resource: "instance",
resourceName: "*",
arnFormat: aws_cdk_lib_1.ArnFormat.SLASH_RESOURCE_NAME,
}),
aws_cdk_lib_1.Stack.of(this).formatArn({
service: "ec2",
resource: "volume",
resourceName: "*",
arnFormat: aws_cdk_lib_1.ArnFormat.SLASH_RESOURCE_NAME,
}),
],
}),
],
}));
}
}

@@ -395,23 +321,2 @@ }

exports.CiStorage = CiStorage;
/**
* Removes leading indentation from all lines of the text.
*/
function dedent(text) {
text = text.replace(/^([ \t\r]*\n)+/s, "").trimEnd();
const matches = text.match(/^[ \t]+/s);
return ((matches ? text.replace(new RegExp("^" + matches[0], "mg"), "") : text) +
"\n");
}
exports.dedent = dedent;
/**
* Converts JS cloud-config representation to yaml user data script.
*/
function yarnDumpCloudConfig(obj) {
return ("#cloud-config\n" +
js_yaml_1.default.dump(obj, {
lineWidth: -1,
quotingType: '"',
styles: { "!!str": "literal" },
}));
}
//# sourceMappingURL=CiStorage.js.map

47

docs/classes/CiStorage.md

@@ -56,3 +56,3 @@ [@clickup/ci-storage-cdk](../README.md) / [Exports](../modules.md) / CiStorage

[src/CiStorage.ts:306](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L306)
[src/CiStorage.ts:161](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L161)

@@ -67,3 +67,3 @@ ## Properties

[src/CiStorage.ts:296](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L296)
[src/CiStorage.ts:150](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L150)

@@ -78,3 +78,3 @@ ___

[src/CiStorage.ts:297](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L297)
[src/CiStorage.ts:151](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L151)

@@ -89,3 +89,3 @@ ___

[src/CiStorage.ts:298](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L298)
[src/CiStorage.ts:152](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L152)

@@ -100,13 +100,20 @@ ___

[src/CiStorage.ts:299](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L299)
[src/CiStorage.ts:153](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L153)
___
### role
### roles
• `Readonly` **role**: `Role`
• `Readonly` **roles**: `Object`
#### Type declaration
| Name | Type |
| :------ | :------ |
| `runner` | `Role` |
| `host` | `Role` |
#### Defined in
[src/CiStorage.ts:300](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L300)
[src/CiStorage.ts:154](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L154)

@@ -121,3 +128,3 @@ ___

[src/CiStorage.ts:301](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L301)
[src/CiStorage.ts:155](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L155)

@@ -132,3 +139,3 @@ ___

[src/CiStorage.ts:302](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L302)
[src/CiStorage.ts:156](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L156)

@@ -143,3 +150,3 @@ ___

[src/CiStorage.ts:303](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L303)
[src/CiStorage.ts:157](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L157)

@@ -154,6 +161,16 @@ ___

[src/CiStorage.ts:304](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L304)
[src/CiStorage.ts:158](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L158)
___
### hostVolumes
• `Readonly` **hostVolumes**: `CfnVolume`[] = `[]`
#### Defined in
[src/CiStorage.ts:159](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L159)
___
### scope

@@ -165,3 +182,3 @@

[src/CiStorage.ts:307](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L307)
[src/CiStorage.ts:162](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L162)

@@ -176,3 +193,3 @@ ___

[src/CiStorage.ts:308](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L308)
[src/CiStorage.ts:163](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L163)

@@ -187,2 +204,2 @@ ___

[src/CiStorage.ts:309](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L309)
[src/CiStorage.ts:164](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L164)

18

docs/interfaces/CiStorageProps.md

@@ -20,3 +20,3 @@ [@clickup/ci-storage-cdk](../README.md) / [Exports](../modules.md) / CiStorageProps

[src/CiStorage.ts:48](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L48)
[src/CiStorage.ts:52](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L52)

@@ -34,3 +34,3 @@ ___

[src/CiStorage.ts:51](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L51)
[src/CiStorage.ts:55](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L55)

@@ -47,3 +47,3 @@ ___

[src/CiStorage.ts:53](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L53)
[src/CiStorage.ts:57](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L57)

@@ -67,3 +67,3 @@ ___

[src/CiStorage.ts:55](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L55)
[src/CiStorage.ts:59](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L59)

@@ -81,3 +81,3 @@ ___

[src/CiStorage.ts:63](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L63)
[src/CiStorage.ts:67](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L67)

@@ -110,3 +110,3 @@ ___

[src/CiStorage.ts:65](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L65)
[src/CiStorage.ts:69](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L69)

@@ -129,3 +129,5 @@ ___

| `imageSsmName` | `string` | SSM parameter name which holds the reference to an instance image. |
| `volumeGb` | `number` | Size of the root volume. |
| `volumeIops` | `number` | IOPS of the docker volume. |
| `volumeThroughput` | `number` | Throughput of the docker volume in MiB/s. |
| `volumeGb` | `number` | Size of the docker volume. |
| `instanceType` | `string` | Full name of the Instance type. |

@@ -136,2 +138,2 @@ | `machines` | `number` | Number of instances to create. |

[src/CiStorage.ts:103](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L103)
[src/CiStorage.ts:107](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L107)

22

docs/modules.md

@@ -12,23 +12,1 @@ [@clickup/ci-storage-cdk](README.md) / Exports

- [CiStorageProps](interfaces/CiStorageProps.md)
## Functions
### dedent
▸ **dedent**(`text`): `string`
Removes leading indentation from all lines of the text.
#### Parameters
| Name | Type |
| :------ | :------ |
| `text` | `string` |
#### Returns
`string`
#### Defined in
[src/CiStorage.ts:561](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L561)

2

package.json
{
"name": "@clickup/ci-storage-cdk",
"description": "A CDK construct to deploy ci-storage infrastructure",
"version": "2.10.291",
"version": "2.10.292",
"license": "MIT",

@@ -6,0 +6,0 @@ "keywords": [

2

src/__tests__/CiStorage.test.ts

@@ -56,2 +56,4 @@ import { App, Duration, Stack } from "aws-cdk-lib";

imageSsmName: "test-imageSsmName",
volumeIops: 3000,
volumeThroughput: 125,
volumeGb: 200,

@@ -58,0 +60,0 @@ instanceType: "t3.large",

379

src/CiStorage.ts

@@ -23,2 +23,3 @@ import { ArnFormat, Duration, Stack, Tags } from "aws-cdk-lib";

SecurityGroup,
CfnVolume,
} from "aws-cdk-lib/aws-ec2";

@@ -28,3 +29,5 @@ import type { RoleProps } from "aws-cdk-lib/aws-iam";

ManagedPolicy,
Policy,
PolicyDocument,
PolicyStatement,
Role,

@@ -37,5 +40,6 @@ ServicePrincipal,

import { Construct } from "constructs";
import yaml from "js-yaml";
import padStart from "lodash/padStart";
import range from "lodash/range";
import { cloudConfigBuild } from "./internal/cloudConfigBuild";
import { cloudConfigYamlDump } from "./internal/cloudConfigYamlDump";
import { namer } from "./internal/namer";

@@ -113,3 +117,7 @@

imageSsmName: string;
/** Size of the root volume. */
/** IOPS of the docker volume. */
volumeIops: number;
/** Throughput of the docker volume in MiB/s. */
volumeThroughput: number;
/** Size of the docker volume. */
volumeGb: number;

@@ -124,156 +132,2 @@ /** Full name of the Instance type. */

/**
* Builds a reusable and never changing cloud config to be passed to the
* instance's CloudInit service.
*/
function buildCloudConfig({
fqdn,
ghTokenSecretName,
ghDockerComposeDirectoryUrl,
keyPairPrivateKeySecretName,
}: {
fqdn: string;
ghTokenSecretName: string;
ghDockerComposeDirectoryUrl: string;
keyPairPrivateKeySecretName: string;
}) {
if (!ghDockerComposeDirectoryUrl.match(/^([^#]+)(?:#([^:]*):(.*))?$/s)) {
throw (
"Cannot parse ghDockerComposeDirectoryUrl. It should be in format: " +
"https://github.com/owner/repo[#[branch]:/directory/with/compose/]"
);
}
const repoUrl = RegExp.$1;
const branch = RegExp.$2 || "";
const path = (RegExp.$3 || ".").replace(/^\/+|\/+$/gs, "");
return {
fqdn: fqdn || undefined,
apt_sources: [
{
source: "deb https://cli.github.com/packages stable main",
keyid: "23F3D4EA75716059",
filename: "github-cli.list",
},
{
source: "deb https://download.docker.com/linux/ubuntu $RELEASE stable",
keyid: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88",
filename: "docker.list",
},
],
packages: [
"awscli",
"gh",
"docker-ce",
"docker-ce-cli",
"containerd.io",
"docker-compose-plugin",
"git",
"gosu",
"mc",
"curl",
"apt-transport-https",
"ca-certificates",
],
write_files: [
{
path: "/etc/sysctl.d/enable-ipv4-forwarding.conf",
content: dedent(`
net.ipv4.conf.all.forwarding=1
`),
},
{
path: "/var/lib/cloud/scripts/per-once/increase-docker-shutdown-timeout.sh",
permissions: "0755",
content: dedent(`
#!/bin/bash
sed -i -E '/TimeoutStartSec=.*/a TimeoutStopSec=3600' /usr/lib/systemd/system/docker.service
systemctl daemon-reload
`),
},
{
path: "/var/lib/cloud/scripts/per-once/switch-ssm-user-to-ubuntu-on-login.sh",
permissions: "0755",
content: dedent(`
#!/bin/bash
sed -i -E '/ExecStart=/i Environment="ENV=/etc/profile.ssm-user"' /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service
echo '[ "$0$@" = "sh" ] && ENV= sudo -u ubuntu -i' > /etc/profile.ssm-user
systemctl daemon-reload
systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true
`),
},
{
path: "/var/lib/cloud/scripts/per-boot/run-docker-compose-on-boot.sh",
permissions: "0755",
content: dedent(`
#!/bin/bash
echo "*/1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh 2>&1 | logger -t run-docker-compose" > /etc/cron.d/run-docker-compose
exec /home/ubuntu/run-docker-compose.sh
`),
},
{
path: "/home/ubuntu/run-docker-compose.sh",
owner: "ubuntu:ubuntu",
permissions: "0755",
defer: true,
content: dedent(`
#!/bin/bash
set -e -o pipefail
# Switch to non-privileged user if running as root.
if [[ $(whoami) != "ubuntu" ]]; then
exec gosu ubuntu:ubuntu "$BASH_SOURCE"
fi
# Ensure there is only one instance of this script running.
exec {FD}<$BASH_SOURCE
flock -n $FD || { echo "Already running."; exit 0; }
# Load private and public keys from Secrets Manager to ~/.ssh.
region=$(ec2metadata --availability-zone | sed "s/[a-z]$//")
mkdir -p ~/.ssh && chmod 700 ~/.ssh
aws secretsmanager get-secret-value --region "$region" \\
--secret-id "${keyPairPrivateKeySecretName}" \\
--query SecretString --output text \\
> ~/.ssh/ci-storage
chmod 600 ~/.ssh/ci-storage
ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub
# Load GitHub PAT from Secrets Manager and login to GitHub.
aws secretsmanager get-secret-value --region "$region" \\
--secret-id "${ghTokenSecretName}" \\
--query SecretString --output text \\
| gh auth login --with-token
gh auth setup-git
# Pull the repository and run docker compose.
mkdir -p ~/git && cd ~/git
if [[ ! -d .git ]]; then
git clone -n --depth=1 --filter=tree:0 ${branch ? `-b "${branch}"` : ""} "${repoUrl}" .
if [[ "${path}" != "." ]]; then
git sparse-checkout set --no-cone "${path}"
fi
git checkout
else
git pull --rebase
fi
sudo usermod -aG docker ubuntu
GH_TOKEN=$(gh auth token) exec sg docker -c 'cd "${path}" && docker compose pull && exec docker compose up --build -d'
`),
},
{
path: "/home/ubuntu/.bash_profile",
owner: "ubuntu:ubuntu",
permissions: "0644",
defer: true,
content: dedent(`
#!/bin/bash
if [ -d ~/git/"${path}" ]; then
cd ~/git/"${path}"
echo '$ docker compose ps'
docker --log-level=ERROR compose ps --format "table {{.Service}}\\t{{.Status}}\\t{{.Ports}}"
echo
fi
`),
},
],
};
}
/**
* A reusable Construct to launch ci-storage infra in some other stack. This

@@ -305,3 +159,3 @@ * class is meant to be put in a public domain and then used in any project.

public readonly keyPairPrivateKeySecretName: string;
public readonly role: Role;
public readonly roles: { runner: Role; host: Role };
public readonly launchTemplate: LaunchTemplate;

@@ -311,2 +165,3 @@ public readonly autoScalingGroup: AutoScalingGroup;

public readonly hostInstances: Instance[] = [];
public readonly hostVolumes: CfnVolume[] = [];

@@ -333,3 +188,3 @@ constructor(

this,
"KeyPair",
namer("key", "pair").pascal,
keyPair.keyPairName,

@@ -341,50 +196,54 @@ );

{
const id = namer("role");
this.role = new Role(this, id.pascal, {
roleName: namer("instance", "profile", "role").pathPascalFrom(this),
assumedBy: new ServicePrincipal("ec2.amazonaws.com"),
managedPolicies: [
ManagedPolicy.fromAwsManagedPolicyName(
"service-role/AmazonEC2RoleforSSM",
),
ManagedPolicy.fromAwsManagedPolicyName("CloudWatchAgentServerPolicy"),
],
inlinePolicies: {
...props.inlinePolicies,
CiStorageKeyPairPolicy: PolicyDocument.fromJson({
Version: "2012-10-17",
Statement: [
{
Effect: "Allow",
Action: ["secretsmanager:GetSecretValue"],
Resource: [
Stack.of(this).formatArn({
service: "secretsmanager",
resource: "secret",
resourceName: `${this.keyPairPrivateKeySecretName}*`,
arnFormat: ArnFormat.COLON_RESOURCE_NAME,
}),
],
},
this.roles = Object.fromEntries(
(["runner", "host"] as const).map((kind) => [
kind,
new Role(this, namer(kind, "role").pascal, {
roleName: namer(kind, "role").pathPascalFrom(this),
assumedBy: new ServicePrincipal("ec2.amazonaws.com"),
managedPolicies: [
ManagedPolicy.fromAwsManagedPolicyName(
"service-role/AmazonEC2RoleforSSM",
),
ManagedPolicy.fromAwsManagedPolicyName(
"CloudWatchAgentServerPolicy",
),
],
inlinePolicies: {
...props.inlinePolicies,
[namer(keyNamer, "key", "pair", "policy").pascal]:
new PolicyDocument({
statements: [
new PolicyStatement({
actions: ["secretsmanager:GetSecretValue"],
resources: [
Stack.of(this).formatArn({
service: "secretsmanager",
resource: "secret",
resourceName: `${this.keyPairPrivateKeySecretName}*`,
arnFormat: ArnFormat.COLON_RESOURCE_NAME,
}),
],
}),
],
}),
[namer(keyNamer, "gh", "token", "policy").pascal]:
new PolicyDocument({
statements: [
new PolicyStatement({
actions: ["secretsmanager:GetSecretValue"],
resources: [
Stack.of(this).formatArn({
service: "secretsmanager",
resource: "secret",
resourceName: `${props.ghTokenSecretName}*`,
arnFormat: ArnFormat.COLON_RESOURCE_NAME,
}),
],
}),
],
}),
},
}),
CiStorageGhTokenPolicy: PolicyDocument.fromJson({
Version: "2012-10-17",
Statement: [
{
Effect: "Allow",
Action: ["secretsmanager:GetSecretValue"],
Resource: [
Stack.of(this).formatArn({
service: "secretsmanager",
resource: "secret",
resourceName: `${props.ghTokenSecretName}*`,
arnFormat: ArnFormat.COLON_RESOURCE_NAME,
}),
],
},
],
}),
},
});
]),
) as typeof this.roles;
}

@@ -403,4 +262,4 @@

const userData = UserData.custom(
yarnDumpCloudConfig(
buildCloudConfig({
cloudConfigYamlDump(
cloudConfigBuild({
fqdn: "",

@@ -421,3 +280,3 @@ ghTokenSecretName: props.ghTokenSecretName,

keyPair: this.keyPair,
role: this.role, // LaunchTemplate creates InstanceProfile internally
role: this.roles.runner, // LaunchTemplate creates InstanceProfile internally
blockDevices: [

@@ -517,5 +376,29 @@ {

: "";
// Unfortunately, there is no way in CDK to auto re-attach the volume to
// an instance if that instance gets replaced. This is because
// CloudFormation first launches a new instance while keeping the old
// instance still running, so the volume can't be attached to the new
// instance - it's already attached to the old one. The solution we use
// here is to do the volume attachment via cloud-config at the new
// instance's initial boot: it first stops the old instance from the new
// one ("aws ec2 stop-instances"), then detaches the volume, and then
// attaches it to the current instance. See logic in
// cloudConfigBuild.ts.
const volumeId = namer(id, "volume");
const volume = new CfnVolume(this, volumeId.pascal, {
availabilityZone: this.vpc.availabilityZones[0],
autoEnableIo: true,
encrypted: true,
iops: props.host.volumeIops,
throughput: props.host.volumeThroughput,
size: props.host.volumeGb,
volumeType: "gp3",
});
Tags.of(volume).add("Name", volumeId.pathKebabFrom(this));
this.hostVolumes.push(volume);
const userData = UserData.custom(
yarnDumpCloudConfig(
buildCloudConfig({
cloudConfigYamlDump(
cloudConfigBuild({
fqdn,

@@ -526,5 +409,7 @@ ghTokenSecretName: props.ghTokenSecretName,

keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName,
mount: { volumeId: volume.attrVolumeId, path: "/mnt" },
}),
),
);
const instance = new Instance(

@@ -536,5 +421,6 @@ this,

securityGroup: this.securityGroup,
availabilityZone: this.vpc.availabilityZones[0],
instanceType: new InstanceType(props.host.instanceType),
machineImage,
role: this.role,
role: this.roles.host,
keyPair: this.keyPair,

@@ -545,3 +431,3 @@ userData,

deviceName: "/dev/sda1",
volume: BlockDeviceVolume.ebs(props.host.volumeGb, {
volume: BlockDeviceVolume.ebs(20, {
encrypted: true,

@@ -569,30 +455,49 @@ volumeType: EbsDeviceVolumeType.GP2,

}
{
const id = namer("host", "volume", "policy");
const conditions = {
StringEquals: {
["ec2:ResourceTag/aws:cloudformation:stack-name"]:
Stack.of(this).stackName,
},
};
this.roles.host.attachInlinePolicy(
new Policy(this, id.pascal, {
policyName: namer(keyNamer, id).pascal,
statements: [
new PolicyStatement({
actions: ["ec2:DescribeVolumes", "ec2:DescribeInstances"],
resources: ["*"],
// Describe* don't support resource-level permissions and
// conditions.
}),
new PolicyStatement({
actions: [
"ec2:StopInstances",
"ec2:DetachVolume",
"ec2:AttachVolume",
],
conditions, // filter by conditions, not by resource ARNs
resources: [
Stack.of(this).formatArn({
service: "ec2",
resource: "instance",
resourceName: "*",
arnFormat: ArnFormat.SLASH_RESOURCE_NAME,
}),
Stack.of(this).formatArn({
service: "ec2",
resource: "volume",
resourceName: "*",
arnFormat: ArnFormat.SLASH_RESOURCE_NAME,
}),
],
}),
],
}),
);
}
}
}
}
/**
* Removes leading indentation from all lines of the text.
*/
export function dedent(text: string): string {
text = text.replace(/^([ \t\r]*\n)+/s, "").trimEnd();
const matches = text.match(/^[ \t]+/s);
return (
(matches ? text.replace(new RegExp("^" + matches[0], "mg"), "") : text) +
"\n"
);
}
/**
* Converts JS cloud-config representation to yaml user data script.
*/
function yarnDumpCloudConfig(obj: object): string {
return (
"#cloud-config\n" +
yaml.dump(obj, {
lineWidth: -1,
quotingType: '"',
styles: { "!!str": "literal" },
})
);
}

4

src/internal/namer.ts

@@ -32,3 +32,5 @@ import compact from "lodash/compact";

const namer = new NamerOrig(
flatten(names.map((name) => (typeof name === "string" ? name : name.parts)))
flatten(
names.map((name) => (typeof name === "string" ? name : name.parts)),
),
) as Namer;

@@ -35,0 +37,0 @@ namer.pathKebabFrom = (scope) =>

dist/CiStorage.d.ts.map

Sorry, the diff of this file is not supported yet

dist/CiStorage.js.map

Sorry, the diff of this file is not supported yet

dist/internal/namer.d.ts.map

Sorry, the diff of this file is not supported yet

dist/internal/namer.js.map

Sorry, the diff of this file is not supported yet

src/__tests__/__snapshots__/CiStorage.test.ts.snap

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc