@dataform/core - npm Package Compare versions

Comparing version 0.0.2-alpha.4 to 0.0.2-alpha.5

build/dataform.d.ts

@@ -7,2 +7,3 @@ import * as protos from "@dataform/protos";
 export declare class Dataform {
+    static ROOT_DIR: string;
     projectConfig: protos.IProjectConfig;
@@ -9,0 +10,0 @@ materializations: {

build/dataform.js

@@ -5,2 +5,3 @@ "use strict";
 const adapters = require("./adapters");
+const utils = require("./utils");
 const materialization_1 = require("./materialization");
@@ -44,2 +45,3 @@ const operation_1 = require("./operation");
         }
+        operation.proto.fileName = utils.getCallerFile(Dataform.ROOT_DIR);
         this.operations[name] = operation;
@@ -56,2 +58,3 @@ return operation;
         }
+        materialization.proto.fileName = utils.getCallerFile(Dataform.ROOT_DIR);
         this.materializations[name] = materialization;
@@ -67,2 +70,3 @@ return materialization;
         }
+        assertion.proto.fileName = utils.getCallerFile(Dataform.ROOT_DIR);
         this.assertions[name] = assertion;
@@ -80,3 +84,4 @@ return assertion;
 }
+Dataform.ROOT_DIR = "";
 exports.Dataform = Dataform;
 //# sourceMappingURL=dataform.js.map

build/materialization.d.ts

@@ -5,2 +5,14 @@ import { Dataform } from "./index";
 export declare type MaterializationType = "table" | "view" | "incremental";
+export interface MConfig {
+    type: MaterializationType;
+    query: MContextable<string>;
+    where: MContextable<string>;
+    pre: MContextable<string | string[]>;
+    post: MContextable<string | string[]>;
+    assert: MContextable<string | string[]>;
+    dependencies: string | string[];
+    schema: {
+        [key: string]: string;
+    } | string[];
+}
 export declare class Materialization {
@@ -14,2 +26,3 @@ proto: protos.Materialization;
     private contextableAssertions;
+    config(config: MConfig): void;
     type(type: MaterializationType): this;
@@ -21,7 +34,8 @@ query(query: MContextable<string>): this;
     assert(query: MContextable<string | string[]>): void;
-    dependency(value: string): this;
-    describe(key: string, description: string): any;
-    describe(map: {
+    dependency(value: string | string[]): this;
+    schema(key: string, description?: string): any;
+    schema(map: {
         [key: string]: string;
     }): any;
+    schema(keys: string[]): any;
     compile(): protos.Materialization;
@@ -40,7 +54,9 @@ }
     assert(query: MContextable<string | string[]>): string;
-    describe(key: string, description: string): any;
-    describe(map: {
+    schema(key: string, description?: string): any;
+    schema(map: {
         [key: string]: string;
     }): any;
+    schema(keys: string[]): any;
+    describe(key: string, description?: string): string;
     apply<T>(value: MContextable<T>): T;
 }

build/materialization.js

@@ -10,3 +10,32 @@ "use strict";
         });
+        this.contextablePres = [];
+        this.contextablePosts = [];
+        this.contextableAssertions = [];
     }
+    config(config) {
+        if (config.type) {
+            this.type(config.type);
+        }
+        if (config.query) {
+            this.query(config.query);
+        }
+        if (config.where) {
+            this.where(config.where);
+        }
+        if (config.pre) {
+            this.pre(config.pre);
+        }
+        if (config.post) {
+            this.post(config.post);
+        }
+        if (config.assert) {
+            this.assert(config.assert);
+        }
+        if (config.dependencies) {
+            this.dependency(config.dependencies);
+        }
+        if (config.schema) {
+            this.schema(config.schema);
+        }
+    }
     type(type) {
@@ -25,25 +54,32 @@ this.proto.type = type;
     pre(pres) {
-        this.contextablePres = pres;
+        this.contextablePres.push(pres);
         return this;
     }
     post(posts) {
-        this.contextablePosts = posts;
+        this.contextablePosts.push(posts);
         return this;
     }
     assert(query) {
-        this.contextableAssertions = query;
+        this.contextableAssertions.push(query);
     }
     dependency(value) {
-        this.proto.dependencies.push(value);
+        this.proto.dependencies = (this.proto.dependencies || []).concat(typeof value === "string" ? [value] : value);
         return this;
     }
-    describe(keyOrMap, description) {
-        if (!!this.proto.descriptions) {
-            this.proto.descriptions = {};
+    schema(keyOrKeysOrMap, description) {
+        if (!!this.proto.schema) {
+            this.proto.schema = {};
         }
-        if (typeof keyOrMap === "string") {
-            this.proto.descriptions[keyOrMap] = description;
+        if (typeof keyOrKeysOrMap === "string") {
+            this.proto.schema[keyOrKeysOrMap] = description || "";
         }
+        else if (keyOrKeysOrMap instanceof Array) {
+            keyOrKeysOrMap.forEach(key => {
+                this.proto.schema[key] = "";
+            });
+        }
         else {
-            Object.assign(this.proto.descriptions, keyOrMap);
+            Object.keys(keyOrKeysOrMap).forEach(key => {
+                this.proto.schema[key] = keyOrKeysOrMap[key] || "";
+            });
         }
@@ -59,25 +95,24 @@ }
         }
-        if (this.contextablePres) {
-            var appliedPres = context.apply(this.contextablePres);
-            this.proto.pres =
-                typeof appliedPres == "string" ? [appliedPres] : appliedPres;
-            this.contextablePres = null;
-        }
-        if (this.contextablePosts) {
-            var appliedPosts = context.apply(this.contextablePosts);
-            this.proto.posts =
-                typeof appliedPosts == "string" ? [appliedPosts] : appliedPosts;
-            this.contextablePosts = null;
-        }
-        if (this.contextableAssertions) {
-            var appliedAssertions = context.apply(this.contextableAssertions);
-            this.proto.assertions =
-                typeof appliedAssertions == "string"
-                    ? [appliedAssertions]
-                    : appliedAssertions;
-            this.contextableAssertions = null;
-        }
+        this.contextablePres.forEach(contextablePres => {
+            var appliedPres = context.apply(contextablePres);
+            this.proto.pres = (this.proto.pres || []).concat(typeof appliedPres == "string" ? [appliedPres] : appliedPres);
+        });
+        this.contextablePres = [];
+        this.contextablePosts.forEach(contextablePosts => {
+            var appliedPosts = context.apply(contextablePosts);
+            this.proto.posts = (this.proto.posts || []).concat(typeof appliedPosts == "string" ? [appliedPosts] : appliedPosts);
+        });
+        this.contextablePosts = [];
+        this.contextableAssertions.forEach(contextableAssertions => {
+            var appliedAssertions = context.apply(contextableAssertions);
+            this.proto.assertions = (this.proto.assertions || []).concat(typeof appliedAssertions == "string"
+                ? [appliedAssertions]
+                : appliedAssertions);
+        });
+        this.contextableAssertions = [];
         try {
             var tree = parser.parse(this.proto.query, {});
-            this.proto.parsedColumns = tree.statement[0].result.map(res => res.alias).map(column => column || "*");
+            this.proto.parsedColumns = tree.statement[0].result
+                .map(res => res.alias)
+                .map(column => column || "*");
         }
@@ -127,9 +162,10 @@ catch (e) {
     }
-    describe(keyOrMap, description) {
-        this.materialization.describe(keyOrMap, description);
-        if (typeof keyOrMap == "string") {
-            return keyOrMap;
-        }
+    schema(keyOrKeysOrMap, description) {
+        this.materialization.schema(keyOrKeysOrMap, description);
         return "";
     }
+    describe(key, description) {
+        this.materialization.schema(key, description);
+        return key;
+    }
     apply(value) {
@@ -136,0 +172,0 @@ if (typeof value === "function") {

build/utils.d.ts

@@ -0,1 +1,2 @@
+export declare function relativePath(path: string, base: string): string;
 export declare function baseFilename(path: string): string;
@@ -5,1 +6,2 @@ export declare function compileSql(code: string, path: string): string;
 export declare function matchPatterns(patterns: string[], values: string[]): string[];
+export declare function getCallerFile(rootDir: string): string;

build/utils.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+function relativePath(path, base) {
+    if (base.length == 0) {
+        return path;
+    }
+    var stripped = path.substr(base.length);
+    if (stripped.startsWith("/")) {
+        return stripped.substr(1);
+    }
+    else {
+        return stripped;
+    }
+}
+exports.relativePath = relativePath;
 function baseFilename(path) {
@@ -41,2 +54,23 @@ var pathSplits = path.split("/");
 exports.matchPatterns = matchPatterns;
+function getCallerFile(rootDir) {
+    var originalFunc = Error.prepareStackTrace;
+    var callerfile;
+    try {
+        var err = new Error();
+        var currentfile;
+        Error.prepareStackTrace = function (err, stack) {
+            return stack;
+        };
+        currentfile = err.stack.shift().getFileName();
+        while (err.stack.length) {
+            callerfile = err.stack.shift().getFileName();
+            if (currentfile !== callerfile && !callerfile.includes("vm2/lib/") && !callerfile.includes("@dataform/core/"))
+                break;
+        }
+    }
+    catch (e) { }
+    Error.prepareStackTrace = originalFunc;
+    return relativePath(callerfile, rootDir);
+}
+exports.getCallerFile = getCallerFile;
 //# sourceMappingURL=utils.js.map

dataform.ts

@@ -9,3 +9,6 @@ import * as protos from "@dataform/protos";
 
+
 export class Dataform {
+  public static ROOT_DIR = "";
+
   projectConfig: protos.IProjectConfig;
@@ -62,2 +65,3 @@
     }
+    operation.proto.fileName = utils.getCallerFile(Dataform.ROOT_DIR);
     // Add it to global index.
@@ -76,2 +80,3 @@ this.operations[name] = operation;
     }
+    materialization.proto.fileName = utils.getCallerFile(Dataform.ROOT_DIR);
     // Add it to global index.
@@ -89,2 +94,3 @@ this.materializations[name] = materialization;
     }
+    assertion.proto.fileName = utils.getCallerFile(Dataform.ROOT_DIR);
     // Add it to global index.
@@ -91,0 +97,0 @@ this.assertions[name] = assertion;

materialization.ts

@@ -8,2 +8,13 @@ import { Dataform } from "./index";
 
+export interface MConfig {
+  type: MaterializationType;
+  query: MContextable<string>;
+  where: MContextable<string>;
+  pre: MContextable<string | string[]>;
+  post: MContextable<string | string[]>;
+  assert: MContextable<string | string[]>;
+  dependencies: string | string[];
+  schema: { [key: string]: string } | string[];
+}
+
 export class Materialization {
@@ -20,6 +31,33 @@ proto: protos.Materialization = protos.Materialization.create({
   private contextableWhere: MContextable<string>;
-  private contextablePres: MContextable<string | string[]>;
-  private contextablePosts: MContextable<string | string[]>;
-  private contextableAssertions: MContextable<string | string[]>;
+  private contextablePres: MContextable<string | string[]>[] = [];
+  private contextablePosts: MContextable<string | string[]>[] = [];
+  private contextableAssertions: MContextable<string | string[]>[] = [];
 
+  public config(config: MConfig) {
+    if (config.type) {
+      this.type(config.type);
+    }
+    if (config.query) {
+      this.query(config.query);
+    }
+    if (config.where) {
+      this.where(config.where);
+    }
+    if (config.pre) {
+      this.pre(config.pre);
+    }
+    if (config.post) {
+      this.post(config.post);
+    }
+    if (config.assert) {
+      this.assert(config.assert);
+    }
+    if (config.dependencies) {
+      this.dependency(config.dependencies);
+    }
+    if (config.schema) {
+      this.schema(config.schema as any);
+    }
+  }
+
   public type(type: MaterializationType) {
@@ -41,3 +79,3 @@ this.proto.type = type;
   public pre(pres: MContextable<string | string[]>) {
-    this.contextablePres = pres;
+    this.contextablePres.push(pres);
     return this;
@@ -47,3 +85,3 @@ }
   public post(posts: MContextable<string | string[]>) {
-    this.contextablePosts = posts;
+    this.contextablePosts.push(posts);
     return this;
@@ -53,23 +91,32 @@ }
   public assert(query: MContextable<string | string[]>) {
-    this.contextableAssertions = query;
+    this.contextableAssertions.push(query);
   }
 
-  public dependency(value: string) {
-    this.proto.dependencies.push(value);
+  public dependency(value: string | string[]) {
+    this.proto.dependencies = (this.proto.dependencies || []).concat(
+      typeof value === "string" ? [value] : value
+    );
     return this;
   }
 
-  public describe(key: string, description: string);
-  public describe(map: { [key: string]: string });
-  public describe(
-    keyOrMap: string | { [key: string]: string },
+  public schema(key: string, description?: string);
+  public schema(map: { [key: string]: string });
+  public schema(keys: string[]);
+  public schema(
+    keyOrKeysOrMap: string | string[] | { [key: string]: string },
     description?: string
   ) {
-    if (!!this.proto.descriptions) {
-      this.proto.descriptions = {};
+    if (!!this.proto.schema) {
+      this.proto.schema = {};
     }
-    if (typeof keyOrMap === "string") {
-      this.proto.descriptions[keyOrMap] = description;
+    if (typeof keyOrKeysOrMap === "string") {
+      this.proto.schema[keyOrKeysOrMap] = description || "";
+    } else if (keyOrKeysOrMap instanceof Array) {
+      keyOrKeysOrMap.forEach(key => {
+        this.proto.schema[key] = "";
+      });
     } else {
-      Object.assign(this.proto.descriptions, keyOrMap);
+      Object.keys(keyOrKeysOrMap).forEach(key => {
+        this.proto.schema[key] = keyOrKeysOrMap[key] || "";
+      });
     }
@@ -89,24 +136,27 @@ }
 
-    if (this.contextablePres) {
-      var appliedPres = context.apply(this.contextablePres);
-      this.proto.pres =
-        typeof appliedPres == "string" ? [appliedPres] : appliedPres;
-      this.contextablePres = null;
-    }
+    this.contextablePres.forEach(contextablePres => {
+      var appliedPres = context.apply(contextablePres);
+      this.proto.pres = (this.proto.pres || []).concat(
+        typeof appliedPres == "string" ? [appliedPres] : appliedPres
+      );
+    });
+    this.contextablePres = [];
 
-    if (this.contextablePosts) {
-      var appliedPosts = context.apply(this.contextablePosts);
-      this.proto.posts =
-        typeof appliedPosts == "string" ? [appliedPosts] : appliedPosts;
-      this.contextablePosts = null;
-    }
+    this.contextablePosts.forEach(contextablePosts => {
+      var appliedPosts = context.apply(contextablePosts);
+      this.proto.posts = (this.proto.posts || []).concat(
+        typeof appliedPosts == "string" ? [appliedPosts] : appliedPosts
+      );
+    });
+    this.contextablePosts = [];
 
-    if (this.contextableAssertions) {
-      var appliedAssertions = context.apply(this.contextableAssertions);
-      this.proto.assertions =
+    this.contextableAssertions.forEach(contextableAssertions => {
+      var appliedAssertions = context.apply(contextableAssertions);
+      this.proto.assertions = (this.proto.assertions || []).concat(
         typeof appliedAssertions == "string"
           ? [appliedAssertions]
-          : appliedAssertions;
-      this.contextableAssertions = null;
-    }
+          : appliedAssertions
+      );
+    });
+    this.contextableAssertions = [];
 
@@ -116,3 +166,5 @@ // Compute columns.
       var tree = parser.parse(this.proto.query, {});
-      this.proto.parsedColumns = tree.statement[0].result.map(res => res.alias).map(column => column || "*");
+      this.proto.parsedColumns = tree.statement[0].result
+        .map(res => res.alias)
+        .map(column => column || "*");
     } catch (e) {
@@ -173,15 +225,18 @@ // There was an exception parsing the columns, ignore.
 
-  public describe(key: string, description: string);
-  public describe(map: { [key: string]: string });
-  public describe(
-    keyOrMap: string | { [key: string]: string },
+  public schema(key: string, description?: string);
+  public schema(map: { [key: string]: string });
+  public schema(keys: string[]);
+  public schema(
+    keyOrKeysOrMap: string | string[] | { [key: string]: string },
     description?: string
   ) {
-    this.materialization.describe(keyOrMap as any, description);
-    if (typeof keyOrMap == "string") {
-      return keyOrMap;
-    }
+    this.materialization.schema(keyOrKeysOrMap as any, description);
     return "";
   }
 
+  public describe(key: string, description?: string) {
+    this.materialization.schema(key, description);
+    return key;
+  }
+
   public apply<T>(value: MContextable<T>): T {
@@ -188,0 +243,0 @@ if (typeof value === "function") {

package.json

 {
   "name": "@dataform/core",
-  "version": "0.0.2-alpha.4",
+  "version": "0.0.2-alpha.5",
   "description": "Dataform core API.",
@@ -8,3 +8,3 @@ "main": "build/index.js",
   "dependencies": {
-    "@dataform/protos": "^0.0.2-alpha.4",
+    "@dataform/protos": "^0.0.2-alpha.5",
     "protobufjs": "^6.8.8"
@@ -15,3 +15,3 @@ },
   },
-  "gitHead": "59230aea41e1a3e4636dcf665d0d3f638e48d51a"
+  "gitHead": "0901e2716693280bf184f2943dab0d670c526ba3"
 }

utils.ts

@@ -0,1 +1,13 @@
+export function relativePath(path: string, base: string) {
+  if (base.length == 0) {
+    return path;
+  }
+  var stripped = path.substr(base.length);
+  if (stripped.startsWith("/")) {
+    return stripped.substr(1);
+  } else {
+    return stripped;
+  }
+}
+
 export function baseFilename(path: string) {
@@ -45,1 +57,21 @@ var pathSplits = path.split("/");
 }
+
+export function getCallerFile(rootDir: string) {
+  var originalFunc = Error.prepareStackTrace;
+  var callerfile;
+  try {
+    var err = new Error();
+    var currentfile;
+    Error.prepareStackTrace = function(err, stack) {
+      return stack;
+    };
+    currentfile = (err.stack as any).shift().getFileName();
+    while (err.stack.length) {
+      callerfile = (err.stack as any).shift().getFileName();
+
+      if (currentfile !== callerfile && !callerfile.includes("vm2/lib/") && ! callerfile.includes("@dataform/core/")) break;
+    }
+  } catch (e) {}
+  Error.prepareStackTrace = originalFunc;
+  return relativePath(callerfile, rootDir);
+}

No alert changes

Improved metrics

Total package byte prevSize

397184

increased by2.05%

Lines of code

7570

increased by2.42%

Dependency changes

• Updated@dataform/protos@^0.0.2-alpha.5