gatsby-source-crownpeak-dxm - npm Package Compare versions

Comparing version 1.0.0 to 1.1.0

gatsby-node.js

@@ -7,26 +7,28 @@ const dotenv = require("dotenv");
 let contentTypesOptions = [];
-let pages = [];
 
-exports.createPages = async ({ graphql, actions }) => {
+exports.createPages = async ({ graphql, actions, getNodesByType }) => {
     const { createPage } = actions;
-    //console.log(`Creating pages`);
-    for (let page of pages) {
-        //console.log(`Page is of type ${page.type}`);
-        const contentType = contentTypesOptions.find(ct => ct.nodeType === page.type)
-        if (contentType && page.slug) {
-            //console.log(`Creating page for ${page.slug}`);
-            createPage({
-                path: page.slug,
-                component: contentType.template,
-                context: {
-                    slug: page.slug,
-                    assetId: page.assetId,
-                },
+    //console.time(`Creating pages`);
+    contentTypesOptions.filter(ct => ct.template).forEach(ct => {
+        const nodes = getNodesByType(ct.nodeType);
+        //console.log(`Node type is ${ct.nodeType}, count is ${nodes.length}`);
+        if (nodes && nodes.length) {
+            nodes.filter(node => node.slug).forEach(node => {
+                //console.log(`Creating page for ${node.slug}`);
+                createPage({
+                    path: node.slug,
+                    component: ct.template,
+                    context: {
+                        slug: node.slug,
+                        assetId: node.assetid,
+                    },
+                });
             });
         }
-    }
+    });
+    //console.timeEnd(`Creating pages`);
 };
 
 exports.sourceNodes = async (
-        { actions, createContentDigest, createNodeId, getNodesByType },
+        { actions, cache, createContentDigest, createNodeId, getNodesByType },
         pluginOptions
@@ -49,4 +51,6 @@ ) => {
 
-    const { createNode } = actions
+    const timestamp = await cache.get(`timestamp`);
 
+    const { createNode, deleteNode, touchNode } = actions
+
     contentTypesOptions = processContentTypeOptions(pluginOptions.contentTypes);
@@ -57,3 +61,3 @@
         // Get all available content types
-        const typesUrl = `${contentLocation}&q=*:*&facet=true&facet.field=custom_s_type&facet.mincount=1&rows=0&echoParams=none&fq=${(pluginOptions.filterQueries || []).join("&fq=")}`;
+        const typesUrl = `${contentLocation}&q=*:*&facet=true&facet.field=custom_s_type&facet.mincount=1&rows=0&echoParams=none&omitHeader=true&fq=${(pluginOptions.filterQueries || []).join("&fq=")}`;
         //console.log(`DEBUG: querying ${typesUrl}`);
@@ -64,6 +68,38 @@ const types = (await (await fetch(typesUrl)).json()).facet_counts.facet_fields.custom_s_type;
     }
+    let cachedContent = [];
+    // touch nodes to ensure they aren't garbage collected
+    //console.time("Checking cached content");
+    contentTypes.forEach(type => getNodesByType(getNodeTypeFromContentType(type)).forEach(node => { 
+        // BUG: touchNode(node) should work here but is not stopping garbage collection
+        //touchNode(node);
+        // using delete node.internal.owner; createNode(node); seems to serve as a workaround
+        delete node.internal.owner; 
+        createNode(node); 
+        //console.log(`Touched node ${node.assetid}, ${node.internal.type}`);
+        cachedContent.push(node);
+    }));
+    //console.timeEnd("Checking cached content");
+    //console.log(`Found cached content ${cachedContent.map(node => node.assetid)}`);
 
-    // TODO: cache results and only query new data
-    // See https://www.gatsbyjs.com/docs/creating-a-source-plugin/
+    if (cachedContent.length) {
+        // Get a complete list of everything available
+        //console.time("Getting existing content ids");
+        const contentIds = await getExistingContent(contentLocation, pluginOptions, contentTypesOptions.map(type => type.name));
+        //console.timeEnd("Getting existing content ids");
+        //console.log(`Found existing content ${contentIds}`);
 
+        // Clear down any cached content that no longer exists
+        //console.time("Deleting old cached content");
+        cachedContent.forEach(node => {
+            const id = node.assetid;
+            //console.log(`Looking for ${id} in ${contentIds}`);
+            if (contentIds.indexOf(id) < 0) {
+                //console.log(`Deleting old cached content ${id}`);
+                deleteNode(node);
+            }
+        });
+        //console.timeEnd("Deleting old cached content");
+    }
+
+    //console.time("Loading new and modified content");
     for (let i = 0, len = contentTypes.length; i < len; i++) {
@@ -74,3 +110,6 @@ const type = contentTypes[i];
 
-        const url = `${contentLocation}&q=custom_s_type:%22${type}%22&rows=1000&fl=id,custom_s_slug,custom_t_json:[json]&echoParams=none&fq=${(pluginOptions.filterQueries || []).join("&fq=")}`;
+        let url = `${contentLocation}&q=custom_s_type:%22${type}%22&rows=10000&fl=id,slug:custom_s_slug,custom_t_json:[json]&echoParams=none&omitHeader=true&fq=${(pluginOptions.filterQueries || []).join("&fq=")}`;
+        if (timestamp) {
+            url += `&fq=custom_dt_modified:[${new Date(timestamp).toISOString()}%20TO%20NOW]`;
+        }
         //console.log(`DEBUG: querying ${url}`);
@@ -90,3 +129,2 @@ const response = await fetch(url);
             //console.log(`Creating node with type ${nodeType}`);
-            const slug = item.custom_s_slug;
             createNode({
@@ -98,3 +136,3 @@ ...json,
                 children: [],
-                slug,
+                slug: item.slug,
                 internal: {
@@ -106,17 +144,26 @@ type: nodeType,
             });
-
-            if (slug) {
-                // These will be used in createPages above
-                pages.push({
-                    type: nodeType,
-                    assetId: item.id,
-                    slug,
-                });
-            }
         });
     };
-    
+    //console.timeEnd("Loading new and modified content");
+    await cache.set(`timestamp`, Date.now());
+
     return;
 }
 
+const getExistingContent = async (contentLocation, pluginOptions, contentTypes) => {
+    const types = contentTypes.map(t => `%22${t}%22`);
+    const CHUNK_SIZE = 10000;
+    let start = 0;
+    let results = [];
+    while (true) {
+        const url = `${contentLocation}&q=custom_s_type:(${types.join("%20OR%20")})&rows=${CHUNK_SIZE}&start=${start}&fl=i:id&echoParams=none&omitHeader=true&fq=${(pluginOptions.filterQueries || []).join("&fq=")}`;
+        //console.log(`Querying ${url}`);
+        const response = (await (await fetch(url)).json()).response;
+        if (!response || !response.numFound || !response.docs || !response.docs.length) break;
+        results.push(response.docs.map(doc => doc.i));
+        start += CHUNK_SIZE;
+    }
+    return [].concat(...results);
+};
+
 const getContentLocation = () => {
@@ -123,0 +170,0 @@ const cwd = process.env.INIT_CWD || require('path').resolve('.');

index.js

 // noop

package.json

 {
   "name": "gatsby-source-crownpeak-dxm",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Gatsby Source Plugin for Crownpeak DXM.",
   "main": "index.js",
+  "repository": "https://github.com/Crownpeak/gatsby-source-crownpeak-dxm",
   "scripts": {
@@ -7,0 +8,0 @@ "test": "echo \"Error: no test specified\" && exit 1",

README.md

@@ -102,2 +102,3 @@ <a href="https://www.crownpeak.com" target="_blank">![Crownpeak Logo](https://github.com/Crownpeak/DXM-React-SDK/raw/master/images/crownpeak-logo.png?raw=true "Crownpeak Logo")</a>
 | 1.0.0         | 2021MAR15     | Initial Release.                   |
+| 1.1.0         | 2021MAR16     | Add support for incremental builds, bug fix. |
 
@@ -104,0 +105,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No repository

Supply chain risk

Package does not have a linked source code repository. Without this field, a package will have no reference to the location of the source code use to generate the package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

19260

increased by24.66%

Number of package files

6

increased by20%

Lines of code

218

increased by40.65%

Number of lines in readme file

127

increased by0.79%

Worsened metrics

Number of medium supply chain risk alerts

4

increased by100%

No dependency changes