@contiamo/dev - npm Package Compare versions

Comparing version 0.5.0-9-gd2986de to 0.5.0-90-gecaab2f

config/auth/providers.json

@@ -10,3 +10,6 @@ {
         "discoveryEndpoint": "https://login.microsoftonline.com/754bc526-d265-4611-8dbb-b3ed9aff0d65/v2.0",
-        "scopes": ["openid", "profile", "email"]
+        "scopes": ["openid", "profile", "email"],
+        "claimMapping": {
+          "email": "preferred_username"
+        }
       }
@@ -13,0 +16,0 @@ }

config/ui-config.js

@@ -5,11 +5,53 @@ window.contiamo = {
   appBase: "http://localhost:9898",
-  pantheon: {
-    enabled: true
+  workbench: {
+    directQueryEnabled: true
   },
-  idp: {
-    enabled: true
+  monitoring: {
+    baseURLs: {
+      tracing: "http://localhost:16686",
+      spark: "http://localhost:4040",
+    },
+    sections: {
+      system: [
+        { name: "Tracing", base: "tracing", path: "/search" },
+        { name: "SMTP", path: "http://localhost:8025" },
+        { name: "PGAdmin", path: "http://localhost:5050" },
+      ],
+      hub: [
+        {
+          name: "Tracing",
+          base: "tracing",
+          path: "/search?limit=20&lookback=1h&service=hub",
+        },
+      ],
+      idp: [
+        {
+          name: "Tracing",
+          base: "tracing",
+          path: "/search?limit=20&lookback=1h&service=idp",
+        },
+      ],
+      pantheon: [
+        {
+          name: "Tracing",
+          base: "tracing",
+          path: "/search?limit=20&lookback=1h&service=pantheon",
+        },
+      ],
+      datastore: [
+        {
+          name: "Tracing",
+          base: "tracing",
+          path: "/search?limit=20&lookback=1h&service=datastore-manager",
+        },
+      ],
+      "datastore-worker": [
+        {
+          name: "Tracing",
+          base: "tracing",
+          path: "/search?limit=20&lookback=1h&service=datastore-ingest-worker",
+        },
+      ],
+    },
   },
-  dataStore: {
-    enabled: true
-  }
 };

index.js

 #!/usr/bin/env node
 
 const program = require("commander");
+const inquirer = require("inquirer");
+
 const path = require("path");
 const fs = require("fs");
+const os = require("os");
+const { spawn, execSync } = require("child_process");
+
 const slash = require("slash");
-const { spawn, execSync } = require("child_process");
 const YAML = require("yaml");
+const Lokka = require("lokka").Lokka;
+const Transport = require("lokka-transport-http").Transport;
 
@@ -15,6 +21,6 @@ /**
  */
-const exec = command =>
-  new Promise(resolve => {
+const exec = (command, opt) =>
+  new Promise((resolve) => {
     const [c, ...args] = command.split(" ");
-    const task = spawn(c, args, { stdio: "inherit" });
+    const task = spawn(c, args, { stdio: "inherit", ...opt });
     task.on("close", resolve);
@@ -27,4 +33,23 @@ });
 
+const restoreScript = slash(
+  path.relative(process.cwd(), path.join(__dirname, "/scripts/restore.sh"))
+);
+
+const snapshotScript = slash(
+  path.relative(process.cwd(), path.join(__dirname, "/scripts/snapshot.sh"))
+);
+
 const dockerCompose = YAML.parse(fs.readFileSync(dockerComposeFile, "utf-8"));
 const services = Object.keys(dockerCompose.services);
+const contiamoServices = Object.entries(dockerCompose.services).reduce(
+  (mem, [name, s]) => {
+    if (!s.image.startsWith("$")) return mem;
+    const detail = /\$\{(\w+):-eu.gcr.io\/dev-and-test-env\/([\w-]+)/.exec(
+      s.image
+    );
+    if (mem.map((m) => m.repo).includes(detail[2])) return mem;
+    return [...mem, { repo: detail[2], envKey: detail[1], name }];
+  },
+  []
+);
 
@@ -37,2 +62,14 @@ // Program
 
+const printStartMessage = () => {
+  console.log("");
+  console.log("Tracing ui: http://localhost:16686/search");
+  console.log("Dev ui:     http://localhost:9898/contiamo/profile");
+  console.log("SMTP ui:    http://localhost:8025");
+  console.log("Spark ui:   http://localhost:4040");
+  console.log("Email:      [e-mail]");
+  console.log("Password:   localdev");
+  console.log("stunnel:    ❌ not configured");
+  console.log("JMX:        port 5001, username pantheon, password localdev");
+};
+
 program.version(version);
@@ -47,3 +84,3 @@
   .command("docker-auth")
-  .description("Setup docker authentification with gcloud")
+  .description("Setup docker authentication with gcloud")
   .action(() => {
@@ -61,3 +98,5 @@ // Doesn't work with `exec` ¯\_(ツ)_/¯
     exec(
-      `docker-compose -f ${dockerComposeFile} pull datastore ui auth pantheon profiler hub graphql`
+      `docker-compose -f ${dockerComposeFile} pull ${contiamoServices
+        .map((i) => i.name)
+        .join(" ")}`
     );
@@ -68,16 +107,35 @@ });
   .command("start")
-  .description("Run the local dev enviromnent on http://localhost:9898")
+  .description("Run the local dev environment on http://localhost:9898")
   .action(() => {
     exec(
       `docker-compose -f ${dockerComposeFile} up -d --force-recreate --remove-orphans`
-    ).then(() => {
-      console.log("-----");
-      console.log("Tracing ui: http://localhost:16686/search");
-      console.log("Dev ui: http://localhost:9898");
-      console.log("Email: [e-mail]");
-      console.log("Password: localdev");
-    });
+    ).then(printStartMessage);
   });
 
 program
+  .command("create-snapshot [snapshotPath]")
+  .description("Create Contiamo dev env snapshot")
+  .action((snapshotPath = "localdev.snapshot") => {
+    exec(`bash ${snapshotScript} ${snapshotPath}`)
+      .then(() => {
+        console.log("");
+        console.log(`${snapshotPath} was created 🥳`);
+      })
+      .catch((e) => {
+        console.error(e);
+      });
+  });
+
+program
+  .command("restore-snapshot [snapshotPath]")
+  .description("Restore Contiamo dev env snapshot")
+  .action((snapshotPath = "localdev.snapshot") => {
+    exec(`bash ${restoreScript} ${snapshotPath}`)
+      .then(printStartMessage)
+      .catch((e) => {
+        console.error(e);
+      });
+  });
+
+program
   .command("logs [SERVICE...]")
@@ -90,7 +148,22 @@ .option(
   .description("Get logs of a service.")
-  .action((_, { parent: { rawArgs }, tail, follow }) => {
+  .action(async (_, { parent: { rawArgs }, tail, follow }) => {
     let args = [];
     if (typeof tail === "string") args.push(`--tail ${tail}`);
     if (follow) args.push(`-f`);
-    args.push(...rawArgs.filter(i => services.includes(i)));
+    const servicesToLog = rawArgs.filter((i) => services.includes(i));
+    if (servicesToLog.length === 0) {
+      const { values } = await inquirer.prompt([
+        {
+          type: "checkbox",
+          name: "values",
+          choices: contiamoServices.map((i) => ({
+            name: i.name,
+            value: i.name,
+          })),
+          message: "Which services do you want to log?",
+        },
+      ]);
+      servicesToLog.push(...values);
+    }
+    args.push(...servicesToLog);
     exec(`docker-compose -f ${dockerComposeFile} logs ${args.join(" ")}`);
@@ -102,5 +175,23 @@ });
   .description("Restart running containers.")
-  .action((_, { parent: { rawArgs } }) => {
-    let args = rawArgs.filter(i => services.includes(i));
-    exec(`docker-compose -f ${dockerComposeFile} restart ${args.join(" ")}`);
+  .action(async (_, { parent: { rawArgs } }) => {
+    let servicesToRestart = rawArgs.filter((i) => services.includes(i));
+    if (servicesToRestart.length === 0) {
+      const { values } = await inquirer.prompt([
+        {
+          type: "checkbox",
+          name: "values",
+          choices: services.map((i) => ({
+            name: i,
+            value: i,
+          })),
+          message: "Which services do you want to restart?",
+        },
+      ]);
+      servicesToRestart.push(...values);
+    }
+    exec(
+      `docker-compose -f ${dockerComposeFile} restart ${servicesToRestart.join(
+        " "
+      )}`
+    );
   });
@@ -130,3 +221,3 @@
         {}
-      )
+      ),
     };
@@ -136,2 +227,119 @@ console.log(YAML.stringify(output));
 
+program
+  .command("pr-preview [pullRequests...]")
+  .description("Testing PR images (ex: `pr-preview hub:123 contiamo-ui:456`)")
+  .action(async (pullRequests) => {
+    const prPreviewableServices = Object.values(dockerCompose.services).reduce(
+      (mem, s) => {
+        if (!s.image.startsWith("$")) return mem;
+        const detail = /\$\{(\w+):-eu.gcr.io\/dev-and-test-env\/([\w-]+)/.exec(
+          s.image
+        );
+        if (mem.map((m) => m.repo).includes(detail[2])) return mem;
+        return [...mem, { repo: detail[2], envKey: detail[1] }];
+      },
+      []
+    );
+
+    if (pullRequests.length === 0) {
+      const { repositories } = await inquirer.prompt([
+        {
+          type: "checkbox",
+          name: "repositories",
+          choices: prPreviewableServices.map((i) => ({
+            name: i.repo,
+            value: i.repo,
+          })),
+          message: "Please select a repository",
+        },
+      ]);
+
+      for (const repo of repositories) {
+        // Retrieve the 10 last github PR
+        let accessToken;
+        const githubTokenPath = path.join(os.homedir(), ".restful-react");
+
+        if (fs.existsSync(githubTokenPath)) {
+          accessToken = fs.readFileSync(githubTokenPath, "utf-8");
+        } else {
+          const answers = await inquirer.prompt([
+            {
+              type: "input",
+              name: "githubToken",
+              message:
+                "Please provide a GitHub token with `repo` rules checked ( https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line )",
+            },
+          ]);
+          fs.writeFileSync(githubTokenPath, answers.githubToken);
+          accessToken = answers.githubToken;
+        }
+
+        const client = new Lokka({
+          transport: new Transport("https://api.github.com/graphql", {
+            headers: {
+              "user-agent": "contiamo-dev",
+              authorization: `bearer ${accessToken}`,
+            },
+          }),
+        });
+
+        const last10prReq = await client
+          .query(
+            `{
+          repository(name:"${
+            repo.startsWith("sync-") ? "sync" : repo
+          }", owner:"contiamo") {
+            pullRequests(last: 10){
+              nodes {
+                number
+                title
+              }
+            }
+          }
+        }`
+          )
+          .then((res) => res.repository.pullRequests.nodes);
+
+        const { pr } = await inquirer.prompt({
+          type: "list",
+          name: "pr",
+          message: `Please select the PR you want to pull for ${repo}`,
+          choices: last10prReq.reverse().map((i) => ({
+            name: `#${i.number} - ${i.title}`,
+            value: i.number,
+          })),
+        });
+
+        pullRequests.push(`${repo}:${pr}`);
+      }
+    }
+
+    if (pullRequests.length === 0) {
+      console.log("You need to select at least one pull-request");
+      return;
+    }
+
+    await Promise.all(
+      pullRequests.map(async (pullRequest) => {
+        const [repo, pr] = pullRequest.split(":");
+        console.log(`Pull image for ${repo} PR #${pr}`);
+        await exec(
+          `docker pull eu.gcr.io/dev-and-test-env/${repo}-preview:pr-${pr}`
+        );
+      })
+    );
+
+    const env = process.env;
+    pullRequests.forEach((pullRequest) => {
+      const [repo, pr] = pullRequest.split(":");
+      env[
+        prPreviewableServices.find((i) => i.repo === repo).envKey
+      ] = `eu.gcr.io/dev-and-test-env/${repo}-preview:pr-${pr}`;
+    });
+
+    exec(`docker-compose -f ${dockerComposeFile} up -d --force-recreate`, {
+      env,
+    });
+  });
+
 program.parse(process.argv);

package.json

 {
   "name": "@contiamo/dev",
-  "version": "0.5.0-9-gd2986de",
-  "description": "Dev enviromnent for contiamo",
+  "version": "0.5.0-90-gecaab2f",
+  "description": "Dev environment for contiamo",
   "bin": {
@@ -27,6 +27,9 @@ "contiamo-dev": "index.js"
   "dependencies": {
-    "commander": "^3.0.1",
+    "commander": "^5.1.0",
+    "inquirer": "^7.3.0",
+    "lokka": "^1.7.0",
+    "lokka-transport-http": "^1.6.1",
     "slash": "^3.0.0",
-    "yaml": "^1.6.0"
+    "yaml": "^1.10.0"
   }
 }

README.md

@@ -69,3 +69,3 @@ # Contiamo Local Dev Environment
 
-### Starting The Environment
+### Starting a fresh environment
 Finally, to start the development environment, run
@@ -85,2 +85,93 @@
 
+### Starting with the latest locadev snapshot
+
+The above section starts with a completely empty environment. A standard development environment
+with preconfigured data sources (the internal metadbs) is provided in the project and can be started
+with
+
+```sh
+make load-snapshot
+```
+
+The existing environment (if any) will be stopped and _destroyed_, so be careful. It will then
+start the db, load the data, and then start the rest of the environment.
+
+The environment
+* contains two users `[e-mail]` and `[e-mail]` both with password `localdev`
+* has all of the datahub metadbs installed, `foodmart`, `alaska`, and `liftdata`
+* there are two virtualdbs with two views each. One that shows the maintenance tasks inside Hub and the other showing the use of PostGIS queries
+* Mr. Lemon is an admin for everything
+* Lemon Jr is not an admin and has various permission levels, `liftdata` is private and not available to Lemon Jr
+* There is a basic amount of metadata assigned to the datasources and tables including custom fields, descriptions, a mix of names, and even one with documentation
+
+This should allow for basic development and testing of most use cases.
+
+
+### Overriding the service images
+The image for each service can be overridden using env variables
+
+| variable              | value                                             |
+|-----------------------|---------------------------------------------------|
+| `AUTH_IMAGE`          | `eu.gcr.io/dev-and-test-env/idp:dev`              |
+| `GRAPHQL_IMAGE`       | `eu.gcr.io/dev-and-test-env/pgql-server:dev`      |
+| `UI_IMAGE`            | `eu.gcr.io/dev-and-test-env/contiamo-ui:dev`      |
+| `HUB_IMAGE`           | `eu.gcr.io/dev-and-test-env/hub:dev`              |
+| `DATASTORE_IMAGE`     | `eu.gcr.io/dev-and-test-env/datastore:dev`        |
+| `PANTHEON_IMAGE`      | `eu.gcr.io/dev-and-test-env/pantheon:dev`         |
+| `HUB_IMAGE`           | `eu.gcr.io/dev-and-test-env/hub:dev`              |
+| `PROFILER_IMAGE`      | `eu.gcr.io/dev-and-test-env/profiler:dev`         |
+| `SYNC_INGESTER_IMAGE` | `eu.gcr.io/dev-and-test-env/sync-ingester:latest` |
+| `SYNC_AGENT_TABLEAU_IMAGE` | `eu.gcr.io/dev-and-test-env/sync-agent-tableau:dev` |
+
+You can manually override the image used by setting the required variable and the restarting the services
+
+```sh
+export HUB_IMAGE=eu.gcr.io/dev-and-test-env/hub:v1.2.3
+make stop start
+```
+
+### Integrations
+The default environment runs only the core services required to support the Data Source integrations. To enable the `demo` sign-up service or integration sync-agents for other resource types (like Tableau), you need to enable the optional integration services. To do this, simply export this env variable
+
+```sh
+export COMPOSE_FILES="-f docker-compose.yml -f docker-compose-extra.yml"
+```
+
+This will modify the `start` and `stop` commands to include the integration services.
+
+### Testing PR images
+A helper make target is provided that will automatically pull and restart the local environment with PR preview image for the specified services.
+
+For example to test PR 501 for `hub` together with PR 489 for `contiamo-ui`, use
+
+```sh
+make pr-preview services=hub:501,contiamo-ui:489
+```
+
+All other services will use the default images.
+
+To reset to the original state, use
+
+```sh
+make stop start
+```
+
+### End-To-End API testing
+The project comes with a suite of end-to-end tests that use the API to verify that the backend services are working as expected. You can run this in any environment by using
+
+```sh
+make test
+```
+
+This assumes that you have already started the localdev environment using `make start` or `make pr-preview`.
+
+
+### Datasets for testing the Profiler
+Two pre-created datasets have been created that provide more interesting stats and entity detection profiles. These should be used to test the Profiler and the related UI components.
+
+The datasets are available in `./datasets`
+
+1. [`pii.csv`](./datasets/pii.csv) contains PII columns that should be detected during the entity detection profile.
+2. [`sales.csv'](./datasets/sales.csv) also contains PII data, but is a good sample for the stats report.
+
 ### Start and add an external data source
@@ -130,5 +221,5 @@ We have a couple of data sets available on GCR for internal testing:
 |------------|---------------------|
-| `HOST`     | `alaska`          |
+| `HOST`     | `alaska`            |
 | `PORT`     | `5432`              |
-| `DATABASE` | `alaska`          |
+| `DATABASE` | `alaska`            |
 | `USER`     | `pantheon`          |
@@ -154,3 +245,3 @@ | `PASS`     | `contiamodatahub19` |
 
-### Adding the metadbs as external data sources
+## Adding the metadbs as external data sources
 You can add the Data Hubs own metadbs to the Data Hub, meaning you can inspect the internals of the Data Hub from the Data Hub :) . Each of the
@@ -167,2 +258,20 @@ following databases can be added as PostgreSQL data sources.
 
+
+## Accessing the metadbs with pgadmin
+
+Go to http://localhost:5050 (The link is on http://localhost:9898/lemonade-shop/configuration page)
+
+Login with the following credentials:
+
+- Email: `[e-mail]`
+- Password: `admin`
+
+Add the `metadb` server with the following connection info:
+
+- Host name/address: `metadb`
+- Port: `5433`
+- Username: `user`
+- Password: `localdev`
+- Save password?: ✅
+
 ## Cleaning up
@@ -192,3 +301,3 @@ If you need to reclaim space or want to restart your environment from scratch use
 ```sh
-./snapshot.sh path/to/my/file.gpg
+./snapshot.sh localdev.snapshot
 ```
@@ -205,3 +314,3 @@
 ```sh
-./restore.sh path/to/the/shapshot.gpg
+./restore.sh localdev.snapshot
 ```
@@ -214,2 +323,4 @@
 
+The `make load-snapshot` uses the committed `localdev.snapshot`.  You can use the script, as described above, to load any other snapsnots
+
 ## Tips
@@ -216,0 +327,0 @@

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

2698449

increased by3111.29%

Number of package files

78

increased by44.44%

Lines of code

463

increased by204.61%

Number of lines in readme file

427

increased by35.13%

Worsened metrics

Dependency count

6

increased by100%

Number of low supply chain risk alerts

2

increased by100%

Number of medium supply chain risk alerts

2

increased by100%

Dependency changes

• + Addedinquirer@^7.3.0

• + Addedlokka@^1.7.0

• + Addedlokka-transport-http@^1.6.1

• + Addedansi-escapes@4.3.2(transitive)

• + Addedansi-regex@5.0.1(transitive)

• + Addedansi-styles@4.3.0(transitive)

• + Addedbabel-runtime@6.26.0(transitive)

• + Addedchalk@4.1.2(transitive)

• + Addedchardet@0.7.0(transitive)

• + Addedcli-cursor@3.1.0(transitive)

• + Addedcli-width@3.0.0(transitive)

• + Addedcolor-convert@2.0.1(transitive)

• + Addedcolor-name@1.1.4(transitive)

• + Addedcommander@5.1.0(transitive)

• + Addedcore-js@2.6.12(transitive)

• + Addedemoji-regex@8.0.0(transitive)

• + Addedencoding@0.1.13(transitive)

• + Addedescape-string-regexp@1.0.5(transitive)

• + Addedexternal-editor@3.1.0(transitive)

• + Addedfigures@3.2.0(transitive)

• + Addedhas-flag@4.0.0(transitive)

• + Addediconv-lite@0.4.240.6.3(transitive)

• + Addedinquirer@7.3.3(transitive)

• + Addedis-fullwidth-code-point@3.0.0(transitive)

• + Addedis-stream@1.1.0(transitive)

• + Addedlodash@4.17.21(transitive)

• + Addedlokka@1.7.0(transitive)

• + Addedlokka-transport-http@1.6.1(transitive)

• + Addedmimic-fn@2.1.0(transitive)

• + Addedmute-stream@0.0.8(transitive)

• + Addednode-fetch@1.7.3(transitive)

• + Addedonetime@5.1.2(transitive)

• + Addedos-tmpdir@1.0.2(transitive)

• + Addedregenerator-runtime@0.11.1(transitive)

• + Addedrestore-cursor@3.1.0(transitive)

• + Addedrun-async@2.4.1(transitive)

• + Addedrxjs@6.6.7(transitive)

• + Addedsafer-buffer@2.1.2(transitive)

• + Addedsignal-exit@3.0.7(transitive)

• + Addedstring-width@4.2.3(transitive)

• + Addedstrip-ansi@6.0.1(transitive)

• + Addedsupports-color@7.2.0(transitive)

• + Addedthrough@2.3.8(transitive)

• + Addedtmp@0.0.33(transitive)

• + Addedtslib@1.14.1(transitive)

• + Addedtype-fest@0.21.3(transitive)

• + Addeduuid@2.0.3(transitive)

• + Addedwhatwg-fetch@1.1.1(transitive)

• - Removedcommander@3.0.2(transitive)

• Updatedcommander@^5.1.0

• Updatedyaml@^1.10.0