@stackbit/sdk - npm Package Compare versions

Comparing version 0.1.15 to 0.1.16

dist/analyzer/analyzer-utils.d.ts

 import { FileBrowser } from './file-browser';
 export declare function findDirsWithPackageDependency(fileBrowser: FileBrowser, packageNames: string[]): Promise<string[]>;
-export declare function extractNodeEnvironmentVariablesFromFile(fileBrowser: FileBrowser, filePath: string): Promise<string[]>;
+export declare function extractNodeEnvironmentVariablesFromFile(data: string): Promise<string[]>;
+interface GatsbySourceFilesystemOptions {
+    name: string;
+    path: string;
+    ignore?: string[];
+}
+export declare function getGatsbySourceFilesystemOptions(data: string): GatsbySourceFilesystemOptions[];
+export declare function getGatsbySourceFilesystemOptionsUsingRegExp(data: string): {
+    name: string;
+    path: string;
+}[];
+export {};

dist/analyzer/analyzer-utils.js

@@ -6,6 +6,8 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractNodeEnvironmentVariablesFromFile = exports.findDirsWithPackageDependency = void 0;
+exports.getGatsbySourceFilesystemOptionsUsingRegExp = exports.getGatsbySourceFilesystemOptions = exports.extractNodeEnvironmentVariablesFromFile = exports.findDirsWithPackageDependency = void 0;
 const path_1 = __importDefault(require("path"));
 const lodash_1 = __importDefault(require("lodash"));
 const utils_1 = require("@stackbit/utils");
+// not sure why, but using import acorn from 'acorn' doesn't work
+const acorn = require('acorn');
 async function findDirsWithPackageDependency(fileBrowser, packageNames) {
@@ -30,8 +32,4 @@ const fileName = 'package.json';
 exports.findDirsWithPackageDependency = findDirsWithPackageDependency;
-async function extractNodeEnvironmentVariablesFromFile(fileBrowser, filePath) {
+async function extractNodeEnvironmentVariablesFromFile(data) {
     const envVars = [];
-    const data = await fileBrowser.getFileData(filePath);
-    if (!data || typeof data !== 'string') {
-        return envVars;
-    }
     const envVarsRe = /process\.env\.(\w+)/g;
@@ -45,2 +43,202 @@ let reResult;
 exports.extractNodeEnvironmentVariablesFromFile = extractNodeEnvironmentVariablesFromFile;
+function getGatsbySourceFilesystemOptions(data) {
+    // Use https://astexplorer.net/ with "JavaScript" and "acorn" parser to generate ESTree
+    const ast = acorn.parse(data, { ecmaVersion: 2020 });
+    // find an object having the following format:
+    // {
+    //   resolve: 'gatsby-source-filesystem',
+    //   options: {
+    //     path: `${__dirname}/content`,
+    //     name: 'pages'
+    //   }
+    // }
+    const result = [];
+    traverseESTree(ast, (node) => {
+        if (!isObjectExpressionNode(node)) {
+            return true;
+        }
+        const resolveProperty = findObjectProperty(node, 'resolve');
+        if (!resolveProperty) {
+            return true;
+        }
+        // we found an object with 'resolve' property, which is one of the plugins
+        // from now on, return false to not continue traversing the current subtree
+        const isGatsbySourceFileSystem = propertyValueEqual(resolveProperty, 'gatsby-source-filesystem');
+        if (!isGatsbySourceFileSystem) {
+            return false;
+        }
+        const optionsProperty = findObjectProperty(node, 'options');
+        if (!optionsProperty || !isObjectExpressionNode(optionsProperty.value)) {
+            return false;
+        }
+        const pathProperty = findObjectProperty(optionsProperty.value, 'path');
+        const nameProperty = findObjectProperty(optionsProperty.value, 'name');
+        if (!pathProperty || !nameProperty) {
+            return false;
+        }
+        let pathValue = getNodeValue(pathProperty.value);
+        const nameValue = getNodeValue(nameProperty.value);
+        if (typeof pathValue !== 'string' || typeof nameValue !== 'string') {
+            return false;
+        }
+        pathValue = pathValue.replace(/^\${__dirname}\//, '');
+        const ignoreProperty = findObjectProperty(optionsProperty.value, 'ignore');
+        let ignoreValue = ignoreProperty ? getNodeValue(ignoreProperty.value) : null;
+        result.push({
+            name: nameValue,
+            path: pathValue,
+            ...(isStringArray(ignoreValue) ? { ignore: ignoreValue } : {})
+        });
+    }, {
+        iteratePrimitives: false
+    });
+    return result;
+}
+exports.getGatsbySourceFilesystemOptions = getGatsbySourceFilesystemOptions;
+function findObjectProperty(node, propertyName) {
+    return lodash_1.default.find(node.properties, (property) => {
+        return isPropertyNode(property) && propertyNameEqual(property, propertyName);
+    });
+}
+function propertyNameEqual(property, propertyName) {
+    // check both identifier and literal properties
+    // { propertyName: '...' } OR { 'propertyName': '...' }
+    return (isIdentifierNode(property.key) && property.key.name === propertyName) || (isLiteralNode(property.key) && property.key.value === propertyName);
+}
+function propertyValueEqual(property, propertyValue) {
+    // check both literal and template literals values
+    // { propertyName: 'propertyValue' } OR { propertyName: `propertyValue` }
+    if (isLiteralNode(property.value) && property.value.value === propertyValue) {
+        return true;
+    }
+    if (isTemplateLiteralNode(property.value)) {
+        const value = property.value;
+        return (value.expressions.length === 0 &&
+            value.quasis.length === 1 &&
+            isTemplateElementNode(value.quasis[0]) &&
+            value.quasis[0].value.raw === propertyValue);
+    }
+    return false;
+}
+/**
+ * This method doesn't serialize every possible ESTree node value. It only
+ * serializes literals, template literals and array expressions needed to
+ * extract simple hard-coded values.
+ *
+ * If this method cannot serialize a value, it returns undefined
+ */
+function getNodeValue(node) {
+    if (isLiteralNode(node)) {
+        return node.value;
+    }
+    else if (isTemplateLiteralNode(node)) {
+        const expressions = node.expressions;
+        const quasis = node.quasis;
+        const sortedNodes = lodash_1.default.sortBy([...expressions, ...quasis], 'start');
+        return lodash_1.default.reduce(sortedNodes, (result, node) => {
+            if (result === undefined) {
+                return result;
+            }
+            if (isTemplateElementNode(node)) {
+                return result + node.value.raw;
+            }
+            else if (isIdentifierNode(node)) {
+                return result + '${' + node.name + '}';
+            }
+            return undefined;
+        }, '');
+    }
+    else if (isArrayExpressionNode(node)) {
+        return lodash_1.default.reduce(node.elements, (result, node) => {
+            if (result === undefined) {
+                return result;
+            }
+            if (node === null) {
+                return undefined;
+            }
+            const value = getNodeValue(node);
+            if (value === undefined) {
+                return value;
+            }
+            result.push(value);
+            return result;
+        }, []);
+    }
+    return undefined;
+}
+function isObjectExpressionNode(node) {
+    return node.type === 'ObjectExpression';
+}
+function isArrayExpressionNode(node) {
+    return node.type === 'ArrayExpression';
+}
+function isIdentifierNode(node) {
+    return node.type === 'Identifier';
+}
+function isLiteralNode(node) {
+    return node.type === 'Literal';
+}
+function isPropertyNode(node) {
+    return node.type === 'Property';
+}
+function isTemplateLiteralNode(node) {
+    return node.type === 'TemplateLiteral';
+}
+function isTemplateElementNode(node) {
+    return node.type === 'TemplateElement';
+}
+function isStringArray(value) {
+    return lodash_1.default.isArray(value) && lodash_1.default.every(value, lodash_1.default.isString);
+}
+function traverseESTree(value, iteratee, options = {}) {
+    const context = lodash_1.default.get(options, 'context');
+    const iterateCollections = lodash_1.default.get(options, 'iterateCollections', true);
+    const iteratePrimitives = lodash_1.default.get(options, 'iteratePrimitives', true);
+    function _traverse(value, keyPath, stack) {
+        const isArrayOrObject = lodash_1.default.isPlainObject(value) || lodash_1.default.isArray(value) || value instanceof acorn.Node;
+        const invokeIteratee = isArrayOrObject ? iterateCollections : iteratePrimitives;
+        let continueTraversing = true;
+        if (invokeIteratee) {
+            continueTraversing = iteratee.call(context, value, keyPath, stack);
+        }
+        if (isArrayOrObject && continueTraversing) {
+            lodash_1.default.forEach(value, (val, key) => {
+                _traverse(val, lodash_1.default.concat(keyPath, key), lodash_1.default.concat(stack, [value]));
+            });
+        }
+    }
+    _traverse(value, [], []);
+}
+function getGatsbySourceFilesystemOptionsUsingRegExp(data) {
+    // {
+    //   resolve: `gatsby-source-filesystem`,
+    //   options: {
+    //     name: 'pages',
+    //     path: `${__dirname}/src/pages`
+    //   }
+    // }
+    const gatsbySourceFilesystemRegExp = /resolve\s*:\s*(['"`])gatsby-source-filesystem\1\s*,\s*options\s*:\s*{\s*(\w+)\s*:\s*(['"`])([^'"`]+)\3\s*,\s*(\w+)\s*:\s*(['"`])([^'"`]+)\6/g;
+    let match;
+    const fileSystemOptions = [];
+    while ((match = gatsbySourceFilesystemRegExp.exec(data)) !== null) {
+        const option1 = match[2];
+        const option2 = match[5];
+        const value1 = match[4];
+        const value2 = match[7];
+        if (option1 === 'name' && option2 === 'path' && value1 && value2) {
+            fileSystemOptions.push({
+                name: value1,
+                path: value2
+            });
+        }
+        else if (option1 === 'path' && option2 === 'name' && value1 && value2) {
+            fileSystemOptions.push({
+                name: value2,
+                path: value1
+            });
+        }
+    }
+    return fileSystemOptions;
+}
+exports.getGatsbySourceFilesystemOptionsUsingRegExp = getGatsbySourceFilesystemOptionsUsingRegExp;
 //# sourceMappingURL=analyzer-utils.js.map

dist/analyzer/schema-generator.js

@@ -20,16 +20,26 @@ "use strict";
 async function generateSchema({ ssgMatchResult, ...fileBrowserOptions }) {
-    var _a, _b, _c;
+    var _a;
     const fileBrowser = file_browser_1.getFileBrowserFromOptions(fileBrowserOptions);
     await fileBrowser.listFiles();
     const ssgDir = (_a = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.ssgDir) !== null && _a !== void 0 ? _a : '';
-    const rootPagesDir = getDir(ssgDir, (_b = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.pagesDir) !== null && _b !== void 0 ? _b : '');
-    const rootDataDir = getDir(ssgDir, (_c = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.dataDir) !== null && _c !== void 0 ? _c : '');
-    const excludedPageFiles = getExcludedPageFiles(rootPagesDir, ssgMatchResult);
-    const excludedDataFiles = getExcludedDataFiles(rootDataDir, ssgMatchResult);
-    // TODO: in some projects, pages can be defined as JSON files as well
-    const pageFiles = await readDirRecursivelyWithFilter(fileBrowser, rootPagesDir, excludedPageFiles, consts_1.MARKDOWN_FILE_EXTENSIONS);
-    const dataFiles = await readDirRecursivelyWithFilter(fileBrowser, rootDataDir, excludedDataFiles, consts_1.DATA_FILE_EXTENSIONS);
+    let pagesDir = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.pagesDir;
+    let dataDir = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.dataDir;
+    const { filePaths: pageFiles, contentDirFromRoot: pagesDirFromRoot } = await listContentFiles({
+        fileBrowser,
+        contentDir: pagesDir,
+        ssgMatchResult,
+        excludedFiles: consts_1.EXCLUDED_MARKDOWN_FILES,
+        allowedExtensions: consts_1.MARKDOWN_FILE_EXTENSIONS
+    });
+    const { filePaths: dataFiles, contentDirFromRoot: dataDirFromRoot } = await listContentFiles({
+        fileBrowser,
+        contentDir: dataDir,
+        ssgMatchResult,
+        excludedFiles: consts_1.EXCLUDED_DATA_FILES,
+        allowedExtensions: consts_1.DATA_FILE_EXTENSIONS,
+        excludedFilesInSSGDir: ['config.*', '_config.*']
+    });
     const pageModelsResults = await generatePageModelsForFiles({
         filePaths: pageFiles,
-        dirPath: rootPagesDir,
+        dirPathFromRoot: pagesDirFromRoot,
         fileBrowser: fileBrowser,
@@ -41,3 +51,3 @@ pageTypeKey: ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.pageTypeKey,
         filePaths: dataFiles,
-        dirPath: rootDataDir,
+        dirPathFromRoot: dataDirFromRoot,
         fileBrowser: fileBrowser,
@@ -48,13 +58,15 @@ objectModels: pageModelsResults.objectModels
     let dataModels = analyzeDataFileMatchingProperties(dataModelsResults.dataModels);
-    let pagesDir = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.pagesDir;
     if (pagesDir === undefined && pageModels.length > 0) {
-        const result = extractLowestCommonAncestorFolderFromModels(pageModels);
-        pagesDir = getDir(ssgDir, result.commonDir);
-        pageModels = result.models;
+        const pagesLCADir = getLowestCommonAncestorFolderFromModels(pageModels);
+        pagesDir = getDir(ssgDir, pagesLCADir);
+        if (pagesLCADir !== '') {
+            pageModels = adjustModelsWithLowestCommonAncestor(pageModels, pagesLCADir);
+        }
     }
-    let dataDir = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.dataDir;
     if (dataDir === undefined && dataModels.length > 0) {
-        const result = extractLowestCommonAncestorFolderFromModels(dataModels);
-        dataDir = getDir(ssgDir, result.commonDir);
-        dataModels = result.models;
+        const dataLCADir = getLowestCommonAncestorFolderFromModels(dataModels);
+        dataDir = getDir(ssgDir, dataLCADir);
+        if (dataLCADir !== '') {
+            dataModels = adjustModelsWithLowestCommonAncestor(dataModels, dataLCADir);
+        }
     }
@@ -82,37 +94,43 @@ const objectModels = lodash_1.default.map(dataModelsResults.objectModels, (objectModel, index) => {
 }
-function getExcludedPageFiles(rootPagesDir, ssgMatchResult) {
-    const excludedPageFiles = [...consts_1.EXCLUDED_COMMON_FILES, ...consts_1.EXCLUDED_MARKDOWN_FILES];
-    if (rootPagesDir === '') {
-        // if pagesDir wasn't specifically set to empty string, ignore content files in the root folder
-        if ((ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.pagesDir) === undefined) {
-            excludedPageFiles.push('*.*');
-        }
-        if (ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.publishDir) {
-            excludedPageFiles.push(ssgMatchResult.publishDir);
-        }
-        if (ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.staticDir) {
-            excludedPageFiles.push(ssgMatchResult.staticDir);
-        }
+async function listContentFiles({ fileBrowser, contentDir, ssgMatchResult, excludedFiles, allowedExtensions, excludedFilesInSSGDir }) {
+    var _a, _b;
+    const ssgDir = (_a = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.ssgDir) !== null && _a !== void 0 ? _a : '';
+    const contentDirs = (_b = ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.contentDirs) !== null && _b !== void 0 ? _b : [];
+    let filePaths;
+    let contentDirFromRoot;
+    if (contentDir !== undefined || contentDirs.length === 0) {
+        contentDirFromRoot = getDir(ssgDir, contentDir !== null && contentDir !== void 0 ? contentDir : '');
+        // TODO: in some projects, pages can be defined as JSON files as well
+        filePaths = await readDirRecursivelyWithFilter({ fileBrowser, contentDir, ssgMatchResult, excludedFiles, allowedExtensions, excludedFilesInSSGDir });
     }
-    return excludedPageFiles;
-}
-function getExcludedDataFiles(rootDataDir, ssgMatchResult) {
-    const excludedDataFiles = [...consts_1.EXCLUDED_COMMON_FILES, ...consts_1.EXCLUDED_DATA_FILES];
-    if (rootDataDir === '') {
-        excludedDataFiles.push('config.*', '_config.*');
-        // if dataDir wasn't specifically set to empty string, ignore content files in the root folder
-        if ((ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.dataDir) === undefined) {
-            excludedDataFiles.push('*.*');
-        }
-        if (ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.publishDir) {
-            excludedDataFiles.push(ssgMatchResult.publishDir);
-        }
-        if (ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.staticDir) {
-            excludedDataFiles.push(ssgMatchResult.staticDir);
-        }
+    else {
+        contentDirFromRoot = ssgDir;
+        filePaths = await utils_1.reducePromise(contentDirs, async (pageFiles, contentDir) => {
+            const files = await readDirRecursivelyWithFilter({
+                fileBrowser,
+                contentDir,
+                ssgMatchResult,
+                excludedFiles,
+                allowedExtensions,
+                excludedFilesInSSGDir,
+                filesRelativeToSSGDir: true
+            });
+            return pageFiles.concat(files);
+        }, []);
     }
-    return excludedDataFiles;
+    return {
+        contentDirFromRoot,
+        filePaths
+    };
 }
-async function readDirRecursivelyWithFilter(fileBrowser, dirPath, excludedFiles, allowedExtensions) {
-    return fileBrowser.readFilesRecursively(dirPath, {
+async function readDirRecursivelyWithFilter(options) {
+    var _a, _b, _c;
+    const excludedFiles = [
+        ...consts_1.EXCLUDED_COMMON_FILES,
+        ...options.excludedFiles,
+        ...getExcludedFiles(options.contentDir, options.excludedFilesInSSGDir, options.ssgMatchResult)
+    ];
+    const ssgDir = (_b = (_a = options.ssgMatchResult) === null || _a === void 0 ? void 0 : _a.ssgDir) !== null && _b !== void 0 ? _b : '';
+    const contentDirFromRoot = getDir(ssgDir, (_c = options.contentDir) !== null && _c !== void 0 ? _c : '');
+    const filePaths = options.fileBrowser.readFilesRecursively(contentDirFromRoot, {
         filter: (fileResult) => {
@@ -126,17 +144,40 @@ if (micromatch_1.default.isMatch(fileResult.filePath, excludedFiles)) {
             const extension = path_1.default.extname(fileResult.filePath).substring(1);
-            return allowedExtensions.includes(extension);
+            return options.allowedExtensions.includes(extension);
         }
     });
+    if (options.filesRelativeToSSGDir) {
+        return lodash_1.default.map(filePaths, (filePath) => { var _a; return path_1.default.join((_a = options.contentDir) !== null && _a !== void 0 ? _a : '', filePath); });
+    }
+    return filePaths;
 }
-async function generatePageModelsForFiles({ filePaths, dirPath, fileBrowser, pageTypeKey, objectModels }) {
+function getExcludedFiles(contentDir, excludedFilesInSSGDir, ssgMatchResult) {
+    const excludedFiles = [];
+    if (contentDir === undefined || contentDir === '') {
+        if (excludedFilesInSSGDir) {
+            excludedFiles.push(...excludedFilesInSSGDir);
+        }
+        // if contentDir (pagesDir or dataDir) wasn't specifically set to empty string, ignore content files in the root folder
+        if (contentDir === undefined) {
+            excludedFiles.push('*.*');
+        }
+        if (ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.publishDir) {
+            excludedFiles.push(ssgMatchResult.publishDir);
+        }
+        if (ssgMatchResult === null || ssgMatchResult === void 0 ? void 0 : ssgMatchResult.staticDir) {
+            excludedFiles.push(ssgMatchResult.staticDir);
+        }
+    }
+    return excludedFiles;
+}
+async function generatePageModelsForFiles({ filePaths, dirPathFromRoot, fileBrowser, pageTypeKey, objectModels }) {
     let pageModels = [];
     let modelNameCounter = 1;
     for (const filePath of filePaths) {
-        const rootFilePath = path_1.default.join(dirPath, filePath);
-        const rootFilePathObject = path_1.default.parse(rootFilePath);
-        let data = await fileBrowser.getFileData(rootFilePath);
-        const extension = rootFilePathObject.ext.substring(1);
+        const filePathFromRoot = path_1.default.join(dirPathFromRoot, filePath);
+        const filePathObjectFromRoot = path_1.default.parse(filePathFromRoot);
+        let data = await fileBrowser.getFileData(filePathFromRoot);
+        const extension = filePathObjectFromRoot.ext.substring(1);
         // don't load plain files from root dir, even though we ignore files such as README.md when reading files,
         // there still might be plain markdown files we don't want to include
-        if (rootFilePathObject.dir === '' && consts_1.MARKDOWN_FILE_EXTENSIONS.includes(extension) && lodash_1.default.get(data, 'frontmatter') === null) {
+        if (filePathObjectFromRoot.dir === '' && consts_1.MARKDOWN_FILE_EXTENSIONS.includes(extension) && lodash_1.default.get(data, 'frontmatter') === null) {
             continue;
@@ -210,7 +251,7 @@ }
 }
-async function generateDataModelsForFiles({ filePaths, dirPath, fileBrowser, objectModels }) {
+async function generateDataModelsForFiles({ filePaths, dirPathFromRoot, fileBrowser, objectModels }) {
     const dataModels = [];
     let modelNameCounter = 1;
     for (const filePath of filePaths) {
-        let data = await fileBrowser.getFileData(path_1.default.join(dirPath, filePath));
+        let data = await fileBrowser.getFileData(path_1.default.join(dirPathFromRoot, filePath));
         const modelName = `data_${modelNameCounter++}`;
@@ -1048,3 +1089,3 @@ if (lodash_1.default.isPlainObject(data)) {
 }
-function extractLowestCommonAncestorFolderFromModels(models) {
+function getLowestCommonAncestorFolderFromModels(models) {
     let commonDir = null;
@@ -1075,21 +1116,17 @@ for (let model of models) {
         }
-        if (commonDir.length === 1 && commonDir[0] === '') {
+        if (commonDir.length === 0 || (commonDir.length === 1 && commonDir[0] === '')) {
             break;
         }
     }
-    const commonDirString = commonDir === null ? '' : commonDir.join(path_1.default.sep);
-    if (commonDirString === '') {
-        return {
-            models,
-            commonDir: ''
-        };
-    }
-    const adjustedModels = lodash_1.default.map(models, (model) => {
+    return commonDir === null ? '' : commonDir.join(path_1.default.sep);
+}
+function adjustModelsWithLowestCommonAncestor(models, lowestCommonAncestorDir) {
+    return lodash_1.default.map(models, (model) => {
         if (model.file) {
             return Object.assign(model, {
-                file: path_1.default.relative(commonDirString, model.file)
+                file: path_1.default.relative(lowestCommonAncestorDir, model.file)
             });
         }
         else {
-            const folder = path_1.default.relative(commonDirString, model.folder);
+            const folder = path_1.default.relative(lowestCommonAncestorDir, model.folder);
             if (folder) {
@@ -1105,6 +1142,2 @@ return Object.assign(model, {
     });
-    return {
-        models: adjustedModels,
-        commonDir: commonDirString
-    };
 }
@@ -1111,0 +1144,0 @@ function findLowestCommonAncestorFolder(filePaths) {

dist/analyzer/ssg-matcher.d.ts

@@ -13,2 +13,3 @@ import { GetFileBrowserOptions } from './file-browser';
     dataDir?: string;
+    contentDirs?: string[];
     envVars?: string[];
@@ -15,0 +16,0 @@ nodeVersion?: string;

dist/analyzer/ssg-matcher.js

@@ -99,6 +99,14 @@ "use strict";
             const gatsbyConfigPath = path_1.default.join(partialMatch.ssgDir, 'gatsby-config.js');
-            const envVars = await analyzer_utils_1.extractNodeEnvironmentVariablesFromFile(fileBrowser, gatsbyConfigPath);
-            if (!lodash_1.default.isEmpty(envVars)) {
-                partialMatch.envVars = envVars;
+            const configData = await fileBrowser.getFileData(gatsbyConfigPath);
+            if (configData && typeof configData === 'string') {
+                // extract env vars from gatsby config
+                const envVars = await analyzer_utils_1.extractNodeEnvironmentVariablesFromFile(configData);
+                if (!lodash_1.default.isEmpty(envVars)) {
+                    partialMatch.envVars = envVars;
+                }
+                // extract gatsby-source-filesystem paths
+                const gatsbySourceFilesystemOptions = analyzer_utils_1.getGatsbySourceFilesystemOptions(configData);
+                partialMatch.contentDirs = lodash_1.default.map(gatsbySourceFilesystemOptions, 'path');
             }
+            // find node version
             const nodeVesion = await matchNodeVersion(fileBrowser, partialMatch);
@@ -105,0 +113,0 @@ if (nodeVesion) {

dist/config/config-loader.d.ts

@@ -1,4 +0,4 @@
-import { StricterUnion } from '../utils';
 import { ConfigValidationError } from './config-validator';
 import { YamlConfigModel, YamlDataModel, YamlObjectModel, YamlPageModel, YamlConfig } from './config-schema';
+import { StricterUnion } from '../utils';
 export declare type BaseModel = {
@@ -5,0 +5,0 @@ name: string;

dist/config/config-loader.js

@@ -14,2 +14,3 @@ "use strict";
 const schema_utils_1 = require("../schema-utils");
+const utils_1 = require("@stackbit/utils");
 async function loadConfig({ dirPath }) {
@@ -57,10 +58,8 @@ let config;
     let config = await loadConfigFromStackbitYaml(dirPath);
-    if (config) {
-        return config;
+    if (!config) {
+        return null;
     }
-    config = await loadConfigFromDotStackbit(dirPath);
-    if (config) {
-        return config;
-    }
-    return null;
+    const models = await loadExternalModels(dirPath, config);
+    config.models = lodash_1.default.assign(models, config.models);
+    return config;
 }
@@ -74,4 +73,38 @@ async function loadConfigFromStackbitYaml(dirPath) {
     const stackbitYaml = await fs_extra_1.default.readFile(stackbitYamlPath);
-    return js_yaml_1.default.load(stackbitYaml.toString('utf8'), { schema: js_yaml_1.default.JSON_SCHEMA });
+    const config = js_yaml_1.default.load(stackbitYaml.toString('utf8'), { schema: js_yaml_1.default.JSON_SCHEMA });
+    if (!config || typeof config !== 'object') {
+        return null;
+    }
+    return config;
 }
+async function loadExternalModels(dirPath, config) {
+    const modelsSource = lodash_1.default.get(config, 'modelsSource', {});
+    const sourceType = lodash_1.default.get(modelsSource, 'type', 'files');
+    if (sourceType === 'files') {
+        const defaultModelDirs = ['node_modules/@stackbit/components/models', '.stackbit/models'];
+        const modelDirs = lodash_1.default.castArray(lodash_1.default.get(modelsSource, 'modelDirs', defaultModelDirs)).map((modelDir) => lodash_1.default.trim(modelDir, '/'));
+        const modelFiles = await utils_1.reducePromise(modelDirs, async (modelFiles, modelDir) => {
+            const absModelsDir = path_1.default.join(dirPath, modelDir);
+            const dirExists = await fs_extra_1.default.pathExists(absModelsDir);
+            if (!dirExists) {
+                return modelFiles;
+            }
+            const files = await readModelFilesFromDir(absModelsDir);
+            return modelFiles.concat(files.map((filePath) => path_1.default.join(modelDir, filePath)));
+        }, []);
+        return utils_1.reducePromise(modelFiles, async (models, modelFile) => {
+            const model = await utils_1.parseFile(path_1.default.join(dirPath, modelFile));
+            models[model.name] = lodash_1.default.omit(model, 'name');
+            return models;
+        }, {});
+    }
+    return null;
+}
+async function readModelFilesFromDir(modelsDir) {
+    return await utils_1.readDirRecursively(modelsDir, {
+        filter: (filePath, stats) => {
+            return stats.isFile() && path_1.default.extname(filePath) === '.yaml';
+        }
+    });
+}
 async function loadConfigFromDotStackbit(dirPath) {
@@ -78,0 +111,0 @@ const stackbitDotPath = path_1.default.join(dirPath, '.stackbit');

dist/config/config-schema.d.ts

@@ -41,2 +41,6 @@ import Joi from 'joi';
 }
+export interface ModelsSource {
+    type: 'files';
+    modelDirs: string[];
+}
 export declare type AssetsModel = StricterUnion<StaticAssetsModal | RelativeAssetsModal>;
@@ -176,2 +180,3 @@ export interface FieldCommonProps {
     logicFields?: LogicField[];
+    modelsSource?: ModelsSource;
     models?: YamlModels;
@@ -178,0 +183,0 @@ }

dist/config/config-schema.js

@@ -101,2 +101,6 @@ "use strict";
 });
+const modelsSourceSchema = joi_1.default.object({
+    type: 'files',
+    modelDirs: joi_1.default.array().items(joi_1.default.string()).required()
+});
 const assetsSchema = joi_1.default.object({
@@ -257,3 +261,3 @@ referenceType: joi_1.default.string().valid('static', 'relative').required(),
 });
-const modelNamePattern = /^[a-z]([a-z0-9_]*[a-z0-9])?$/;
+const modelNamePattern = /^[a-zA-Z]([a-zA-Z0-9_]*[a-zA-Z0-9])?$/;
 const modelNamePatternMatchErrorCode = 'model.name.pattern.match';
@@ -296,3 +300,3 @@ const modelFileExclusiveErrorCode = 'model.file.only';
     messages: {
-        [modelNamePatternMatchErrorCode]: 'Invalid model name "{{#key}}" at "{{#label}}". A model name must contain only lower case alphanumeric characters and underscores, must start with a lower case letter, and end with alphanumeric character.',
+        [modelNamePatternMatchErrorCode]: 'Invalid model name "{{#key}}" at "{{#label}}". A model name must contain only alphanumeric characters and underscores, must start with a letter, and end with alphanumeric character.',
         [modelFileExclusiveErrorCode]: '{{#label}} cannot be used with "file"',
@@ -327,2 +331,3 @@ [modelIsListItemsRequiredErrorCode]: '{{#label}} is required when "isList" is true',
     logicFields: joi_1.default.array().items(logicField),
+    modelsSource: modelsSourceSchema,
     models: modelsSchema
@@ -329,0 +334,0 @@ })

package.json

 {
     "name": "@stackbit/sdk",
-    "version": "0.1.15",
+    "version": "0.1.16",
     "description": "Stackbit SDK",
@@ -42,2 +42,3 @@ "main": "dist/index.js",
         "@stackbit/utils": "^0.2.0",
+        "acorn": "^8.2.4",
         "fs-extra": "^9.1.0",
@@ -52,2 +53,3 @@ "joi": "^17.4.0",
     "devDependencies": {
+        "@types/estree": "0.0.47",
         "@types/fs-extra": "^9.0.7",
@@ -54,0 +56,0 @@ "@types/js-yaml": "^4.0.0",

src/analyzer/analyzer-utils.ts

 import path from 'path';
 import _ from 'lodash';
 import { reducePromise } from '@stackbit/utils';
+import estree from 'estree';
 
+// not sure why, but using import acorn from 'acorn' doesn't work
+const acorn = require('acorn');
+
 import { FileBrowser } from './file-browser';
@@ -34,8 +38,4 @@
 
-export async function extractNodeEnvironmentVariablesFromFile(fileBrowser: FileBrowser, filePath: string): Promise<string[]> {
+export async function extractNodeEnvironmentVariablesFromFile(data: string): Promise<string[]> {
     const envVars: string[] = [];
-    const data = await fileBrowser.getFileData(filePath);
-    if (!data || typeof data !== 'string') {
-        return envVars;
-    }
     const envVarsRe = /process\.env\.(\w+)/g;
@@ -48,1 +48,237 @@ let reResult;
 }
+
+interface GatsbySourceFilesystemOptions {
+    name: string;
+    path: string;
+    ignore?: string[];
+}
+
+export function getGatsbySourceFilesystemOptions(data: string): GatsbySourceFilesystemOptions[] {
+    // Use https://astexplorer.net/ with "JavaScript" and "acorn" parser to generate ESTree
+    const ast = (acorn.parse(data, { ecmaVersion: 2020 }) as unknown) as estree.Program;
+    // find an object having the following format:
+    // {
+    //   resolve: 'gatsby-source-filesystem',
+    //   options: {
+    //     path: `${__dirname}/content`,
+    //     name: 'pages'
+    //   }
+    // }
+    const result: GatsbySourceFilesystemOptions[] = [];
+    traverseESTree(
+        ast,
+        (node: estree.Node) => {
+            if (!isObjectExpressionNode(node)) {
+                return true;
+            }
+            const resolveProperty = findObjectProperty(node, 'resolve');
+            if (!resolveProperty) {
+                return true;
+            }
+            // we found an object with 'resolve' property, which is one of the plugins
+            // from now on, return false to not continue traversing the current subtree
+
+            const isGatsbySourceFileSystem = propertyValueEqual(resolveProperty, 'gatsby-source-filesystem');
+            if (!isGatsbySourceFileSystem) {
+                return false;
+            }
+            const optionsProperty = findObjectProperty(node, 'options');
+            if (!optionsProperty || !isObjectExpressionNode(optionsProperty.value)) {
+                return false;
+            }
+            const pathProperty = findObjectProperty(optionsProperty.value, 'path');
+            const nameProperty = findObjectProperty(optionsProperty.value, 'name');
+            if (!pathProperty || !nameProperty) {
+                return false;
+            }
+            let pathValue = getNodeValue(pathProperty.value);
+            const nameValue = getNodeValue(nameProperty.value);
+            if (typeof pathValue !== 'string' || typeof nameValue !== 'string') {
+                return false;
+            }
+            pathValue = pathValue.replace(/^\${__dirname}\//, '');
+            const ignoreProperty = findObjectProperty(optionsProperty.value, 'ignore');
+            let ignoreValue = ignoreProperty ? getNodeValue(ignoreProperty.value) : null;
+            result.push({
+                name: nameValue,
+                path: pathValue,
+                ...(isStringArray(ignoreValue) ? { ignore: ignoreValue } : {})
+            });
+        },
+        {
+            iteratePrimitives: false
+        }
+    );
+    return result;
+}
+
+function findObjectProperty(node: estree.ObjectExpression, propertyName: string) {
+    return _.find(node.properties, (property): property is estree.Property => {
+        return isPropertyNode(property) && propertyNameEqual(property, propertyName);
+    });
+}
+
+function propertyNameEqual(property: estree.Property, propertyName: string) {
+    // check both identifier and literal properties
+    // { propertyName: '...' } OR { 'propertyName': '...' }
+    return (isIdentifierNode(property.key) && property.key.name === propertyName) || (isLiteralNode(property.key) && property.key.value === propertyName);
+}
+
+function propertyValueEqual(property: estree.Property, propertyValue: string) {
+    // check both literal and template literals values
+    // { propertyName: 'propertyValue' } OR { propertyName: `propertyValue` }
+    if (isLiteralNode(property.value) && property.value.value === propertyValue) {
+        return true;
+    }
+    if (isTemplateLiteralNode(property.value)) {
+        const value = property.value;
+        return (
+            value.expressions.length === 0 &&
+            value.quasis.length === 1 &&
+            isTemplateElementNode(value.quasis[0]!) &&
+            value.quasis[0]!.value.raw === propertyValue
+        );
+    }
+    return false;
+}
+
+/**
+ * This method doesn't serialize every possible ESTree node value. It only
+ * serializes literals, template literals and array expressions needed to
+ * extract simple hard-coded values.
+ *
+ * If this method cannot serialize a value, it returns undefined
+ */
+function getNodeValue(node: estree.Node): any {
+    if (isLiteralNode(node)) {
+        return node.value;
+    } else if (isTemplateLiteralNode(node)) {
+        const expressions = node.expressions;
+        const quasis = node.quasis;
+        const sortedNodes = _.sortBy([...expressions, ...quasis], 'start');
+        return _.reduce(
+            sortedNodes,
+            (result: string | undefined, node) => {
+                if (result === undefined) {
+                    return result;
+                }
+                if (isTemplateElementNode(node)) {
+                    return result + node.value.raw;
+                } else if (isIdentifierNode(node)) {
+                    return result + '${' + node.name + '}';
+                }
+                return undefined;
+            },
+            ''
+        );
+    } else if (isArrayExpressionNode(node)) {
+        return _.reduce(
+            node.elements,
+            (result: any[] | undefined, node) => {
+                if (result === undefined) {
+                    return result;
+                }
+                if (node === null) {
+                    return undefined;
+                }
+                const value = getNodeValue(node);
+                if (value === undefined) {
+                    return value;
+                }
+                result.push(value);
+                return result;
+            },
+            []
+        );
+    }
+    return undefined;
+}
+
+function isObjectExpressionNode(node: estree.Node): node is estree.ObjectExpression {
+    return node.type === 'ObjectExpression';
+}
+
+function isArrayExpressionNode(node: estree.Node): node is estree.ArrayExpression {
+    return node.type === 'ArrayExpression';
+}
+
+function isIdentifierNode(node: estree.Node): node is estree.Identifier {
+    return node.type === 'Identifier';
+}
+
+function isLiteralNode(node: estree.Node): node is estree.Literal {
+    return node.type === 'Literal';
+}
+
+function isPropertyNode(node: estree.Node): node is estree.Property {
+    return node.type === 'Property';
+}
+
+function isTemplateLiteralNode(node: estree.Node): node is estree.TemplateLiteral {
+    return node.type === 'TemplateLiteral';
+}
+
+function isTemplateElementNode(node: estree.Node): node is estree.TemplateElement {
+    return node.type === 'TemplateElement';
+}
+
+function isStringArray(value: any): value is string[] {
+    return _.isArray(value) && _.every(value, _.isString);
+}
+
+function traverseESTree(
+    value: any,
+    iteratee: (value: any, keyPath: (string | number)[], stack: any[]) => any,
+    options: { iterateCollections?: boolean; iteratePrimitives?: boolean } = {}
+) {
+    const context = _.get(options, 'context');
+    const iterateCollections = _.get(options, 'iterateCollections', true);
+    const iteratePrimitives = _.get(options, 'iteratePrimitives', true);
+
+    function _traverse(value: any, keyPath: (string | number)[], stack: any[]) {
+        const isArrayOrObject = _.isPlainObject(value) || _.isArray(value) || value instanceof acorn.Node;
+        const invokeIteratee = isArrayOrObject ? iterateCollections : iteratePrimitives;
+        let continueTraversing = true;
+        if (invokeIteratee) {
+            continueTraversing = iteratee.call(context, value, keyPath, stack);
+        }
+        if (isArrayOrObject && continueTraversing) {
+            _.forEach(value, (val: any, key: string | number) => {
+                _traverse(val, _.concat(keyPath, key), _.concat(stack, [value]));
+            });
+        }
+    }
+
+    _traverse(value, [], []);
+}
+
+export function getGatsbySourceFilesystemOptionsUsingRegExp(data: string) {
+    // {
+    //   resolve: `gatsby-source-filesystem`,
+    //   options: {
+    //     name: 'pages',
+    //     path: `${__dirname}/src/pages`
+    //   }
+    // }
+    const gatsbySourceFilesystemRegExp = /resolve\s*:\s*(['"`])gatsby-source-filesystem\1\s*,\s*options\s*:\s*{\s*(\w+)\s*:\s*(['"`])([^'"`]+)\3\s*,\s*(\w+)\s*:\s*(['"`])([^'"`]+)\6/g;
+    let match: RegExpExecArray | null;
+    const fileSystemOptions = [];
+    while ((match = gatsbySourceFilesystemRegExp.exec(data)) !== null) {
+        const option1 = match[2];
+        const option2 = match[5];
+        const value1 = match[4];
+        const value2 = match[7];
+        if (option1 === 'name' && option2 === 'path' && value1 && value2) {
+            fileSystemOptions.push({
+                name: value1,
+                path: value2
+            });
+        } else if (option1 === 'path' && option2 === 'name' && value1 && value2) {
+            fileSystemOptions.push({
+                name: value2,
+                path: value1
+            });
+        }
+    }
+    return fileSystemOptions;
+}

src/analyzer/schema-generator.ts

@@ -5,3 +5,3 @@ import path from 'path';
 import moment from 'moment';
-import { append } from '@stackbit/utils';
+import { append, reducePromise } from '@stackbit/utils';
 
@@ -50,15 +50,25 @@ import { FileBrowser, FileResult, getFileBrowserFromOptions, GetFileBrowserOptions } from './file-browser';
     const ssgDir = ssgMatchResult?.ssgDir ?? '';
-    const rootPagesDir = getDir(ssgDir, ssgMatchResult?.pagesDir ?? '');
-    const rootDataDir = getDir(ssgDir, ssgMatchResult?.dataDir ?? '');
+    let pagesDir = ssgMatchResult?.pagesDir;
+    let dataDir = ssgMatchResult?.dataDir;
 
-    const excludedPageFiles = getExcludedPageFiles(rootPagesDir, ssgMatchResult);
-    const excludedDataFiles = getExcludedDataFiles(rootDataDir, ssgMatchResult);
+    const { filePaths: pageFiles, contentDirFromRoot: pagesDirFromRoot } = await listContentFiles({
+        fileBrowser,
+        contentDir: pagesDir,
+        ssgMatchResult,
+        excludedFiles: EXCLUDED_MARKDOWN_FILES,
+        allowedExtensions: MARKDOWN_FILE_EXTENSIONS
+    });
 
-    // TODO: in some projects, pages can be defined as JSON files as well
-    const pageFiles = await readDirRecursivelyWithFilter(fileBrowser, rootPagesDir, excludedPageFiles, MARKDOWN_FILE_EXTENSIONS);
-    const dataFiles = await readDirRecursivelyWithFilter(fileBrowser, rootDataDir, excludedDataFiles, DATA_FILE_EXTENSIONS);
+    const { filePaths: dataFiles, contentDirFromRoot: dataDirFromRoot } = await listContentFiles({
+        fileBrowser,
+        contentDir: dataDir,
+        ssgMatchResult,
+        excludedFiles: EXCLUDED_DATA_FILES,
+        allowedExtensions: DATA_FILE_EXTENSIONS,
+        excludedFilesInSSGDir: ['config.*', '_config.*']
+    });
 
     const pageModelsResults = await generatePageModelsForFiles({
         filePaths: pageFiles,
-        dirPath: rootPagesDir,
+        dirPathFromRoot: pagesDirFromRoot,
         fileBrowser: fileBrowser,
@@ -70,3 +80,3 @@ pageTypeKey: ssgMatchResult?.pageTypeKey,
         filePaths: dataFiles,
-        dirPath: rootDataDir,
+        dirPathFromRoot: dataDirFromRoot,
         fileBrowser: fileBrowser,
@@ -79,13 +89,15 @@ objectModels: pageModelsResults.objectModels
 
-    let pagesDir = ssgMatchResult?.pagesDir;
     if (pagesDir === undefined && pageModels.length > 0) {
-        const result = extractLowestCommonAncestorFolderFromModels(pageModels);
-        pagesDir = getDir(ssgDir, result.commonDir);
-        pageModels = result.models;
+        const pagesLCADir = getLowestCommonAncestorFolderFromModels(pageModels);
+        pagesDir = getDir(ssgDir, pagesLCADir);
+        if (pagesLCADir !== '') {
+            pageModels = adjustModelsWithLowestCommonAncestor(pageModels, pagesLCADir);
+        }
     }
-    let dataDir = ssgMatchResult?.dataDir;
     if (dataDir === undefined && dataModels.length > 0) {
-        const result = extractLowestCommonAncestorFolderFromModels(dataModels);
-        dataDir = getDir(ssgDir, result.commonDir);
-        dataModels = result.models;
+        const dataLCADir = getLowestCommonAncestorFolderFromModels(dataModels);
+        dataDir = getDir(ssgDir, dataLCADir);
+        if (dataLCADir !== '') {
+            dataModels = adjustModelsWithLowestCommonAncestor(dataModels, dataLCADir);
+        }
     }
@@ -120,39 +132,64 @@
 
-function getExcludedPageFiles(rootPagesDir: string, ssgMatchResult: SSGMatchResult | null): string[] {
-    const excludedPageFiles = [...EXCLUDED_COMMON_FILES, ...EXCLUDED_MARKDOWN_FILES];
-    if (rootPagesDir === '') {
-        // if pagesDir wasn't specifically set to empty string, ignore content files in the root folder
-        if (ssgMatchResult?.pagesDir === undefined) {
-            excludedPageFiles.push('*.*');
-        }
-        if (ssgMatchResult?.publishDir) {
-            excludedPageFiles.push(ssgMatchResult.publishDir);
-        }
-        if (ssgMatchResult?.staticDir) {
-            excludedPageFiles.push(ssgMatchResult.staticDir);
-        }
-    }
-    return excludedPageFiles;
+interface ListContentFilesOption {
+    fileBrowser: FileBrowser;
+    contentDir: string | undefined;
+    ssgMatchResult: SSGMatchResult | null;
+    excludedFiles: string[];
+    allowedExtensions: string[];
+    excludedFilesInSSGDir?: string[];
 }
 
-function getExcludedDataFiles(rootDataDir: string, ssgMatchResult: SSGMatchResult | null): string[] {
-    const excludedDataFiles = [...EXCLUDED_COMMON_FILES, ...EXCLUDED_DATA_FILES];
-    if (rootDataDir === '') {
-        excludedDataFiles.push('config.*', '_config.*');
-        // if dataDir wasn't specifically set to empty string, ignore content files in the root folder
-        if (ssgMatchResult?.dataDir === undefined) {
-            excludedDataFiles.push('*.*');
-        }
-        if (ssgMatchResult?.publishDir) {
-            excludedDataFiles.push(ssgMatchResult.publishDir);
-        }
-        if (ssgMatchResult?.staticDir) {
-            excludedDataFiles.push(ssgMatchResult.staticDir);
-        }
+async function listContentFiles({ fileBrowser, contentDir, ssgMatchResult, excludedFiles, allowedExtensions, excludedFilesInSSGDir }: ListContentFilesOption) {
+    const ssgDir = ssgMatchResult?.ssgDir ?? '';
+    const contentDirs = ssgMatchResult?.contentDirs ?? [];
+    let filePaths: string[];
+    let contentDirFromRoot;
+    if (contentDir !== undefined || contentDirs.length === 0) {
+        contentDirFromRoot = getDir(ssgDir, contentDir ?? '');
+        // TODO: in some projects, pages can be defined as JSON files as well
+        filePaths = await readDirRecursivelyWithFilter({ fileBrowser, contentDir, ssgMatchResult, excludedFiles, allowedExtensions, excludedFilesInSSGDir });
+    } else {
+        contentDirFromRoot = ssgDir;
+        filePaths = await reducePromise(
+            contentDirs,
+            async (pageFiles: string[], contentDir) => {
+                const files = await readDirRecursivelyWithFilter({
+                    fileBrowser,
+                    contentDir,
+                    ssgMatchResult,
+                    excludedFiles,
+                    allowedExtensions,
+                    excludedFilesInSSGDir,
+                    filesRelativeToSSGDir: true
+                });
+                return pageFiles.concat(files);
+            },
+            []
+        );
     }
-    return excludedDataFiles;
+    return {
+        contentDirFromRoot,
+        filePaths
+    };
 }
 
-async function readDirRecursivelyWithFilter(fileBrowser: FileBrowser, dirPath: string, excludedFiles: string[], allowedExtensions: string[]) {
-    return fileBrowser.readFilesRecursively(dirPath, {
+interface ReadDirOptions {
+    fileBrowser: FileBrowser;
+    contentDir: string | undefined;
+    ssgMatchResult: SSGMatchResult | null;
+    excludedFiles: string[];
+    allowedExtensions: string[];
+    excludedFilesInSSGDir?: string[];
+    filesRelativeToSSGDir?: boolean;
+}
+
+async function readDirRecursivelyWithFilter(options: ReadDirOptions) {
+    const excludedFiles = [
+        ...EXCLUDED_COMMON_FILES,
+        ...options.excludedFiles,
+        ...getExcludedFiles(options.contentDir, options.excludedFilesInSSGDir, options.ssgMatchResult)
+    ];
+    const ssgDir = options.ssgMatchResult?.ssgDir ?? '';
+    const contentDirFromRoot = getDir(ssgDir, options.contentDir ?? '');
+    const filePaths = options.fileBrowser.readFilesRecursively(contentDirFromRoot, {
         filter: (fileResult: FileResult) => {
@@ -166,10 +203,34 @@ if (micromatch.isMatch(fileResult.filePath, excludedFiles)) {
             const extension = path.extname(fileResult.filePath).substring(1);
-            return allowedExtensions.includes(extension);
+            return options.allowedExtensions.includes(extension);
         }
     });
+    if (options.filesRelativeToSSGDir) {
+        return _.map(filePaths, (filePath) => path.join(options.contentDir ?? '', filePath));
+    }
+    return filePaths;
 }
 
+function getExcludedFiles(contentDir: string | undefined, excludedFilesInSSGDir: string[] | undefined, ssgMatchResult: SSGMatchResult | null): string[] {
+    const excludedFiles = [];
+    if (contentDir === undefined || contentDir === '') {
+        if (excludedFilesInSSGDir) {
+            excludedFiles.push(...excludedFilesInSSGDir);
+        }
+        // if contentDir (pagesDir or dataDir) wasn't specifically set to empty string, ignore content files in the root folder
+        if (contentDir === undefined) {
+            excludedFiles.push('*.*');
+        }
+        if (ssgMatchResult?.publishDir) {
+            excludedFiles.push(ssgMatchResult.publishDir);
+        }
+        if (ssgMatchResult?.staticDir) {
+            excludedFiles.push(ssgMatchResult.staticDir);
+        }
+    }
+    return excludedFiles;
+}
+
 interface GeneratePageModelsOptions {
     filePaths: string[];
-    dirPath: string;
+    dirPathFromRoot: string;
     fileBrowser: FileBrowser;
@@ -181,3 +242,3 @@ pageTypeKey?: string;
     filePaths,
-    dirPath,
+    dirPathFromRoot,
     fileBrowser,
@@ -191,9 +252,9 @@ pageTypeKey,
     for (const filePath of filePaths) {
-        const rootFilePath = path.join(dirPath, filePath);
-        const rootFilePathObject = path.parse(rootFilePath);
-        let data = await fileBrowser.getFileData(rootFilePath);
-        const extension = rootFilePathObject.ext.substring(1);
+        const filePathFromRoot = path.join(dirPathFromRoot, filePath);
+        const filePathObjectFromRoot = path.parse(filePathFromRoot);
+        let data = await fileBrowser.getFileData(filePathFromRoot);
+        const extension = filePathObjectFromRoot.ext.substring(1);
         // don't load plain files from root dir, even though we ignore files such as README.md when reading files,
         // there still might be plain markdown files we don't want to include
-        if (rootFilePathObject.dir === '' && MARKDOWN_FILE_EXTENSIONS.includes(extension) && _.get(data, 'frontmatter') === null) {
+        if (filePathObjectFromRoot.dir === '' && MARKDOWN_FILE_EXTENSIONS.includes(extension) && _.get(data, 'frontmatter') === null) {
             continue;
@@ -281,3 +342,3 @@ }
     filePaths: string[];
-    dirPath: string;
+    dirPathFromRoot: string;
     fileBrowser: FileBrowser;
@@ -289,3 +350,3 @@ objectModels: PartialObjectModel[];
     filePaths,
-    dirPath,
+    dirPathFromRoot,
     fileBrowser,
@@ -298,3 +359,3 @@ objectModels
     for (const filePath of filePaths) {
-        let data = await fileBrowser.getFileData(path.join(dirPath, filePath));
+        let data = await fileBrowser.getFileData(path.join(dirPathFromRoot, filePath));
         const modelName = `data_${modelNameCounter++}`;
@@ -1205,3 +1266,3 @@ if (_.isPlainObject(data)) {
 
-function extractLowestCommonAncestorFolderFromModels<T extends PageModel | DataModel>(models: T[]) {
+function getLowestCommonAncestorFolderFromModels<T extends PageModel | DataModel>(models: T[]): string {
     let commonDir: null | string[] = null;
@@ -1229,14 +1290,11 @@ for (let model of models) {
         }
-        if (commonDir.length === 1 && commonDir[0] === '') {
+        if (commonDir.length === 0 || (commonDir.length === 1 && commonDir[0] === '')) {
             break;
         }
     }
-    const commonDirString = commonDir === null ? '' : commonDir.join(path.sep);
-    if (commonDirString === '') {
-        return {
-            models,
-            commonDir: ''
-        };
-    }
-    const adjustedModels = _.map(
+    return commonDir === null ? '' : commonDir.join(path.sep);
+}
+
+function adjustModelsWithLowestCommonAncestor<T extends PageModel | DataModel>(models: T[], lowestCommonAncestorDir: string) {
+    return _.map(
         models,
@@ -1246,6 +1304,6 @@ (model): T => {
                 return Object.assign(model, {
-                    file: path.relative(commonDirString, model.file)
+                    file: path.relative(lowestCommonAncestorDir, model.file)
                 });
             } else {
-                const folder = path.relative(commonDirString, model.folder!);
+                const folder = path.relative(lowestCommonAncestorDir, model.folder!);
                 if (folder) {
@@ -1261,6 +1319,2 @@ return Object.assign(model, {
     );
-    return {
-        models: adjustedModels,
-        commonDir: commonDirString
-    };
 }
@@ -1267,0 +1321,0 @@

src/analyzer/ssg-matcher.ts

@@ -6,3 +6,3 @@ import path from 'path';
 import { FileBrowser, getFileBrowserFromOptions, GetFileBrowserOptions } from './file-browser';
-import { extractNodeEnvironmentVariablesFromFile, findDirsWithPackageDependency } from './analyzer-utils';
+import { extractNodeEnvironmentVariablesFromFile, findDirsWithPackageDependency, getGatsbySourceFilesystemOptions } from './analyzer-utils';
 import { Config } from '../config/config-loader';
@@ -22,2 +22,3 @@
     dataDir?: string;
+    contentDirs?: string[];
     envVars?: string[];
@@ -132,6 +133,16 @@ nodeVersion?: string;
             const gatsbyConfigPath = path.join(partialMatch.ssgDir, 'gatsby-config.js');
-            const envVars = await extractNodeEnvironmentVariablesFromFile(fileBrowser, gatsbyConfigPath);
-            if (!_.isEmpty(envVars)) {
-                partialMatch.envVars = envVars;
+            const configData = await fileBrowser.getFileData(gatsbyConfigPath);
+            if (configData && typeof configData === 'string') {
+                // extract env vars from gatsby config
+                const envVars = await extractNodeEnvironmentVariablesFromFile(configData);
+                if (!_.isEmpty(envVars)) {
+                    partialMatch.envVars = envVars;
+                }
+
+                // extract gatsby-source-filesystem paths
+                const gatsbySourceFilesystemOptions = getGatsbySourceFilesystemOptions(configData);
+                partialMatch.contentDirs = _.map(gatsbySourceFilesystemOptions, 'path');
             }
+
+            // find node version
             const nodeVesion = await matchNodeVersion(fileBrowser, partialMatch);
@@ -148,2 +159,3 @@ if (nodeVesion) {
             }
+
             return partialMatch;
@@ -150,0 +162,0 @@ }

src/config/config-loader.ts

@@ -6,7 +6,8 @@ import path from 'path';
 import { extendModels, iterateModelFieldsRecursively, isListField } from '@stackbit/schema';
-import { StricterUnion } from '../utils';
 
 import { validate, ConfigValidationResult, ConfigValidationError } from './config-validator';
 import { Field, YamlConfigModel, YamlDataModel, YamlModel, YamlObjectModel, YamlPageModel, YamlConfig } from './config-schema';
+import { StricterUnion } from '../utils';
 import { isPageModel } from '../schema-utils';
+import { parseFile, readDirRecursively, reducePromise } from '@stackbit/utils';
 
@@ -95,13 +96,11 @@ export type BaseModel = {
     let config = await loadConfigFromStackbitYaml(dirPath);
-    if (config) {
-        return config;
+    if (!config) {
+        return null;
     }
-    config = await loadConfigFromDotStackbit(dirPath);
-    if (config) {
-        return config;
-    }
-    return null;
+    const models = await loadExternalModels(dirPath, config);
+    config.models = _.assign(models, config.models);
+    return config;
 }
 
-async function loadConfigFromStackbitYaml(dirPath: string) {
+async function loadConfigFromStackbitYaml(dirPath: string): Promise<any> {
     const stackbitYamlPath = path.join(dirPath, 'stackbit.yaml');
@@ -113,5 +112,49 @@ const stackbitYamlExists = await fse.pathExists(stackbitYamlPath);
     const stackbitYaml = await fse.readFile(stackbitYamlPath);
-    return yaml.load(stackbitYaml.toString('utf8'), { schema: yaml.JSON_SCHEMA });
+    const config = yaml.load(stackbitYaml.toString('utf8'), { schema: yaml.JSON_SCHEMA });
+    if (!config || typeof config !== 'object') {
+        return null;
+    }
+    return config;
 }
 
+async function loadExternalModels(dirPath: string, config: any) {
+    const modelsSource = _.get(config, 'modelsSource', {});
+    const sourceType = _.get(modelsSource, 'type', 'files');
+    if (sourceType === 'files') {
+        const defaultModelDirs = ['node_modules/@stackbit/components/models', '.stackbit/models'];
+        const modelDirs = _.castArray(_.get(modelsSource, 'modelDirs', defaultModelDirs)).map((modelDir: string) => _.trim(modelDir, '/'));
+        const modelFiles = await reducePromise(
+            modelDirs,
+            async (modelFiles: string[], modelDir) => {
+                const absModelsDir = path.join(dirPath, modelDir);
+                const dirExists = await fse.pathExists(absModelsDir);
+                if (!dirExists) {
+                    return modelFiles;
+                }
+                const files = await readModelFilesFromDir(absModelsDir);
+                return modelFiles.concat(files.map((filePath) => path.join(modelDir, filePath)));
+            },
+            []
+        );
+        return reducePromise(
+            modelFiles,
+            async (models: any, modelFile) => {
+                const model = await parseFile(path.join(dirPath, modelFile));
+                models[model.name] = _.omit(model, 'name');
+                return models;
+            },
+            {}
+        );
+    }
+    return null;
+}
+
+async function readModelFilesFromDir(modelsDir: string) {
+    return await readDirRecursively(modelsDir, {
+        filter: (filePath, stats) => {
+            return stats.isFile() && path.extname(filePath) === '.yaml';
+        }
+    });
+}
+
 async function loadConfigFromDotStackbit(dirPath: string) {
@@ -118,0 +161,0 @@ const stackbitDotPath = path.join(dirPath, '.stackbit');

src/config/config-schema.ts

@@ -152,2 +152,12 @@ import Joi from 'joi';
 
+export interface ModelsSource {
+    type: 'files';
+    modelDirs: string[];
+}
+
+const modelsSourceSchema = Joi.object<ModelsSource>({
+    type: 'files',
+    modelDirs: Joi.array().items(Joi.string()).required()
+});
+
 export type AssetsModel = StricterUnion<StaticAssetsModal | RelativeAssetsModal>;
@@ -492,3 +502,3 @@
 
-const modelNamePattern = /^[a-z]([a-z0-9_]*[a-z0-9])?$/;
+const modelNamePattern = /^[a-zA-Z]([a-zA-Z0-9_]*[a-zA-Z0-9])?$/;
 const modelNamePatternMatchErrorCode = 'model.name.pattern.match';
@@ -532,3 +542,3 @@ const modelFileExclusiveErrorCode = 'model.file.only';
             [modelNamePatternMatchErrorCode]:
-                'Invalid model name "{{#key}}" at "{{#label}}". A model name must contain only lower case alphanumeric characters and underscores, must start with a lower case letter, and end with alphanumeric character.',
+                'Invalid model name "{{#key}}" at "{{#label}}". A model name must contain only alphanumeric characters and underscores, must start with a letter, and end with alphanumeric character.',
             [modelFileExclusiveErrorCode]: '{{#label}} cannot be used with "file"',
@@ -564,2 +574,3 @@ [modelIsListItemsRequiredErrorCode]: '{{#label}} is required when "isList" is true',
     logicFields?: LogicField[];
+    modelsSource?: ModelsSource;
     models?: YamlModels;
@@ -587,2 +598,3 @@ }
     logicFields: Joi.array().items(logicField),
+    modelsSource: modelsSourceSchema,
     models: modelsSchema
@@ -589,0 +601,0 @@ })

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

483209

increased by11.95%

Lines of code

8781

increased by12.35%

Worsened metrics

Dependency count

11

increased by10%

Dev dependency count

11

increased by10%

Number of package files

86

decreased by-2.27%

Dependency changes

• + Addedacorn@^8.2.4

• + Addedacorn@8.14.0(transitive)