sourcebit - npm Package Compare versions

Comparing version 0.1.1 to 0.2.0

bin/sourcebit.js

 #!/usr/bin/env node
+require("dotenv").config();
 const mri = require("mri");
 const sourcebit = require("../index");
+const path = require("path");
+
 const { _: method, ...parameters } = mri(process.argv.slice(2));
+const configPath = path.resolve(
+  process.cwd(),
+  parameters.config || "sourcebit.js"
+);
+const config = require(configPath);
 
-sourcebit.setParameters(parameters);
-sourcebit[method]();
+sourcebit[method](config, parameters);

index.js

@@ -1,27 +0,21 @@
-const path = require("path");
 const Sourcebit = require("./lib/sourcebit");
 
-const instance = new Sourcebit();
-
-module.exports.fetch = config => {
-  // If `config` isn't present, we'll look for a `sourcebit.js` file.
+module.exports.fetch = async (config, runtimeParameters) => {
   if (!config) {
-    const filePath = path.join(process.cwd(), "sourcebit.js");
-
-    try {
-      config = require(filePath);
-    } catch (error) {
-      console.log(
-        "ERROR: Could not find a valid `sourcebit.js` configuration file."
-      );
-
-      process.exit(1);
-    }
+    throw new Error(
+      "ERROR: Could not find a valid `sourcebit.js` configuration file."
+    );
   }
 
+  const instance = new Sourcebit({ runtimeParameters });
   const { plugins = [] } = config;
 
-  return instance.runBootstrap(plugins);
+  instance.loadContextFromCache();
+  instance.loadPlugins(plugins);
+
+  await instance.bootstrapAll();
+
+  return instance.transform();
 };
 
-module.exports.setParameters = parameters => instance.setParameters(parameters);
+module.exports.Sourcebit = Sourcebit;

lib/sourcebit.js

@@ -6,105 +6,105 @@ const fs = require("fs");
   constructor({
-    cacheFile = path.join(process.cwd(), ".sourcebit-cache.json")
+    cacheFile = path.join(process.cwd(), ".sourcebit-cache.json"),
+    runtimeParameters = {}
   } = {}) {
     this.cacheFilePath = cacheFile;
     this.context = {};
-    this.parameters = {};
-    this.plugins;
+    this.pluginBlocks = [];
+    this.pluginModules = {};
+    this.runtimeParameters = runtimeParameters;
   }
 
-  getPluginContext(namespace) {
-    return this.context[namespace] || {};
-  }
+  async bootstrapAll() {
+    let queue = Promise.resolve();
 
-  loadCache() {
-    try {
-      const cache = require(this.cacheFilePath);
+    this.pluginBlocks.forEach((_, pluginIndex) => {
+      queue = queue.then(() => this.bootstrapPluginAtIndex(pluginIndex));
+    });
 
-      console.log("Found cache file");
+    await queue;
 
-      return cache;
-    } catch (error) {
-      console.log("Cache file not found");
+    this.saveContextToCache();
 
-      return {};
-    }
+    return this.context;
   }
 
-  loadPlugins(pluginsBlock) {
-    const plugins = {};
+  async bootstrapPluginAtIndex(index) {
+    const pluginBlock = this.pluginBlocks[index];
+    const { options } = pluginBlock;
+    const plugin = this.pluginModules[index];
+    const pluginName = this.getNameOfPluginAtIndex(index);
+    const { defaults, overrides } = this.parsePluginOptions(plugin);
 
-    pluginsBlock.forEach(plugin => {
-      try {
-        plugins[plugin.name] = require(plugin.name);
-      } catch (error) {
-        console.log(`ERROR: Could not load plugin ${plugin.name}`);
+    if (typeof plugin.bootstrap === "function") {
+      await plugin.bootstrap({
+        context: this.context,
+        getPluginContext: this.getContextForNamespace.bind(this, pluginName),
+        log: this.log.bind(this, pluginName),
+        options: Object.assign({}, defaults, options, overrides),
+        refresh: this.transform.bind(this),
+        setPluginContext: this.setContextForNamespace.bind(this, pluginName)
+      });
+    }
 
-        process.exit(1);
-      }
-    });
+    pluginBlock._isBootstrapped = true;
+  }
 
-    return plugins;
+  getContext() {
+    return this.context;
   }
 
-  log(namespace, message) {
-    console.log(`[${namespace}] ${message}`);
+  getContextForNamespace(namespace) {
+    return this.context[namespace] || {};
   }
 
-  async runBootstrap(plugins) {
-    const pluginModules = this.loadPlugins(plugins);
+  getNameOfPluginAtIndex(index) {
+    return this.pluginModules[index].name || `plugin_${index}`;
+  }
 
-    // Attempt to load context from cache.
-    this.context = this.loadCache();
+  loadContextFromCache() {
+    try {
+      this.context = require(this.cacheFilePath);
+    } catch (error) {
+      this.context = {};
+    }
+  }
 
-    let queue = Promise.resolve();
+  loadPlugins(plugins) {
+    this.pluginBlocks = plugins;
 
-    plugins.forEach(({ name, options }) => {
-      queue = queue.then(() => {
-        const plugin = pluginModules[name];
-
-        if (typeof plugin.bootstrap === "function") {
-          return plugin.bootstrap({
-            context: this.context,
-            getPluginContext: this.getPluginContext.bind(this, name),
-            log: this.log.bind(this, name),
-            options,
-            refresh: this.runTransform.bind(this, plugins),
-            setPluginContext: this.setContext.bind(this, name)
-          });
-        }
-      });
+    plugins.forEach((plugin, index) => {
+      this.pluginModules[index] = plugin.module;
     });
+  }
 
-    await queue;
+  log(namespace, message) {
+    if (this.runtimeParameters.quiet) {
+      return;
+    }
 
-    this.saveContextToCache();
-
-    return this.context;
+    console.log(`[${namespace}]`, message);
   }
 
-  runTransform(plugins) {
-    const pluginModules = this.loadPlugins(plugins);
-    const objects = [];
+  parsePluginOptions(plugin) {
+    const { options } = plugin;
 
-    let queue = Promise.resolve(objects);
+    if (!options) return {};
 
-    plugins.forEach(({ name, options }) => {
-      queue = queue.then(objects => {
-        const plugin = pluginModules[name];
+    const defaults = {};
+    const overrides = {};
 
-        if (typeof plugin.transform === "function") {
-          return plugin.transform({
-            context: this.context,
-            getPluginContext: this.getPluginContext.bind(this, name),
-            log: this.log.bind(this, name),
-            objects,
-            options
-          });
-        }
+    Object.keys(options).forEach(key => {
+      if (options[key].default !== undefined) {
+        defaults[key] = options[key].default;
+      }
 
-        return objects;
-      });
+      if (
+        typeof options[key].runtimeParameter === "string" &&
+        this.runtimeParameters[options[key].runtimeParameter] !== undefined
+      ) {
+        overrides[key] = this.runtimeParameters[options[key].runtimeParameter];
+      }
     });
 
-    return queue;
+    return { defaults, overrides };
   }
@@ -122,11 +122,46 @@
 
-  setContext(namespace, data) {
+  setContextForNamespace(namespace, data) {
     this.context[namespace] = data;
   }
 
-  setParameters(parameters) {
-    this.parameters = parameters;
+  setOptionsForPluginAtIndex(index, options) {
+    this.pluginBlocks[index].options = options;
   }
+
+  transform() {
+    const data = {
+      models: [],
+      objects: []
+    };
+
+    let queue = Promise.resolve(data);
+
+    this.pluginBlocks.forEach(({ _isBootstrapped, options }, index) => {
+      if (!_isBootstrapped) return data;
+
+      queue = queue.then(data => {
+        const plugin = this.pluginModules[index];
+        const pluginName = this.getNameOfPluginAtIndex(index);
+        const { defaults, overrides } = this.parsePluginOptions(plugin);
+
+        if (typeof plugin.transform === "function") {
+          return plugin.transform({
+            data,
+            getPluginContext: this.getContextForNamespace.bind(
+              this,
+              pluginName
+            ),
+            log: this.log.bind(this, pluginName),
+            options: Object.assign({}, defaults, options, overrides)
+          });
+        }
+
+        return data;
+      });
+    });
+
+    return queue;
+  }
 }
 
 module.exports = Sourcebit;

package.json

 {
   "name": "sourcebit",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Sourcebit helps developers build data-driven JAMstack sites by pulling data from any third-party resource",
@@ -10,2 +10,3 @@ "main": "index.js",
   "dependencies": {
+    "dotenv": "^8.2.0",
     "mri": "^1.1.4"
@@ -12,0 +13,0 @@ },

README.md

@@ -6,1 +6,127 @@ # Sourcebit
 > Sourcebit helps developers build data-driven JAMstack sites by pulling data from any third-party resource.
+
+## Introduction
+
+Sourcebit connects to multiple data sources using modular components called _source plugins_. These are responsible for fetching data, normalizing it to a standard format, and placing the resulting entries on sets of data called _data buckets_.
+
+Subsequently, any combination of plugins may consume, transform and persist these data buckets in any way they like.
+
+A specific group of plugins, called _target plugins_, is tasked with writing data into a format and location that other programs – such as static site generators – expect.
+
+```
+           +----------------+  +---------------+  +-----------------+
+           |                |  |               |  |                 |
+           |   Contentful   |  |    DatoCMS    |  |     Airtable    |
+           |                |  |               |  |                 |
+           +--------\-------+  +-------|-------+  +--------/--------+
+                     \                 |                  /
+                      \                |                 /
+                       \               |                /
+                        \              |               /
+          +--------------|-------------|--------------|-------------+
+          |              |             |              |             |
+          |        +-----|-----+ +-----------+  +-----|-----+       |
+        S |        | (Plugin)  | | (Plugin)  |  | (Plugin)  |       | S
+        O |        |           | |           |  |           |       | O
+        U |        +-----|-----+ +-----|-----+  +-----|-----+       | U
+        R |              |             |              |             | R
+        C |              |             |              |             | C
+        E |        +-----|-----+ +-----|-----+  +-----|-----+       | E
+        B |        | (Plugin)  | | (Plugin)  |  | (Plugin)  |       | B
+        I |        |           | |           |  |           |       | I
+        T |        +-----|-----+ +-----|-----+  +-----|-----+       | T
+          |              |             |              |             |
+          +--------------|-------------|--------------|-------------+
+                        /              |               \
+                       /               |                \
+                      /                |                 \
+                     /                 |                  \
+                    /                  |                   \
+          +--------/--------+ +--------|-------+   +----------------+
+          |                 | |                |   |                |
+          |     Next.js     | |     Jekyll     |   |      Hugo      |
+          |                 | |                |   |                |
+          +-----------------+ +----------------+   +----------------+
+
+```
+
+## Installation
+
+Sourcebit is distributed as an [npm module](https://www.npmjs.com/package/sourcebit). To install it and add it as a dependency to your project, run:
+
+```
+npm install sourcebit --save
+```
+
+## Configuration
+
+Everything about Sourcebit, including its plugins and their parameters, is configured in a JavaScript object that typically lives in a file called `sourcebit.js`.
+
+It looks something like this:
+
+```js
+module.exports = {
+  plugins: [
+    {
+      module: require("sourcebit-some-plugin-1"),
+      options: {
+        pluginOption1: "foo",
+        pluginOptino2: "bar"
+      }
+    },
+    {
+      module: require("sourcebit-some-plugin-2"),
+      options: {
+        pluginFunction1: (a, b) => a + b
+      }
+    }
+  ]
+};
+```
+
+It's important to note that while every plugin block is defined with the `module` and `options` properties, the actual options passed to each options block varies widely and is defined (and should be documented) by each plugin.
+
+### Interactive setup process
+
+To ease the process of configuring Sourcebit, we've created a command-line tool that provides an interactive setup process. By asking you a series of questions defined by each plugin, this tool generates a `sourcebit.js` file so that you can get up and running in no time.
+
+To start this process, run `npx sourcebit` in your project directory.
+
+## Usage
+
+### As a CommonJS module
+
+To use Sourcebit as a CommonJS module, include it in your project and call its `fetch` method.
+
+```js
+const sourcebit = require("sourcebit");
+const sourcebitConfig = require("./sourcebit.js");
+
+sourcebit.fetch(sourcebitConfig).then(data => {
+  console.log(data);
+});
+```
+
+### As a command-line tool
+
+To use Sourcebit as a command-line tool, run the `sourcebit fetch` command in your project directory.
+
+```
+$ sourcebit fetch
+```
+
+## Plugins
+
+### Source plugins
+
+- [`sourcebit-source-contentful`](http://npmjs.com/package/sourcebit-source-contentful): A source plugin for the [Contentful](https://www.contentful.com/) headless CMS.
+
+### Target plugins
+
+- [`sourcebit-target-jekyll`](http://npmjs.com/package/sourcebit-target-jekyll): A target plugin for the [Jekyll](https://www.jekyllrb.com/) static site generator.
+
+### Other plugins
+
+- [`sourcebit-plugin-content-mapper`](http://npmjs.com/package/sourcebit-plugin-content-mapper): A plugin for creating different data buckets from content models.
+
+> 🚀 If you're interested in creating your own plugin, you might want to check out our [sample plugin repository](https://github.com/stackbithq/sourcebit-sample-plugin), which contains a basic plugin with a fully-annotated source.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

11737

increased by129.15%

Lines of code

157

increased by24.6%

Number of lines in readme file

132

increased by2100%

Number of low supply chain risk alerts

3

decreased by-25%

Worsened metrics

Dependency count

2

increased by100%

Dependency changes

• + Addeddotenv@^8.2.0

• + Addeddotenv@8.6.0(transitive)