contentlayer-source-notion - npm Package Compare versions

Comparing version 0.0.1-alpha.16 to 0.0.1-alpha.17

dist/fetchData/fetchAllDocuments.d.ts

@@ -7,6 +7,7 @@ import type { DataCache } from '@contentlayer/core';
     databaseTypeDefs: DatabaseTypeDef[];
+    previousCache: core.DataCache.Cache | undefined;
     schemaDef: core.SchemaDef;
     options: core.PluginOptions;
 };
-export declare const fetchAllDocuments: ({ databaseTypeDefs, schemaDef, options }: FetchAllDocumentsArgs) => T.Effect<import("@contentlayer/utils/effect").Has<import("@notionhq/client").Client> & OT.HasTracer & import("@contentlayer/utils/effect").Has<import("@notion-render/client").NotionRenderer> & import("@contentlayer/utils/effect").Has<OT.Tracer>, core.SourceFetchDataError, {
+export declare const fetchAllDocuments: ({ databaseTypeDefs, previousCache, schemaDef, options }: FetchAllDocumentsArgs) => T.Effect<import("@contentlayer/utils/effect").Has<import("@notionhq/client").Client> & OT.HasTracer & import("@contentlayer/utils/effect").Has<import("@notion-render/client").NotionRenderer> & import("@contentlayer/utils/effect").Has<OT.Tracer>, core.SourceFetchDataError, {
     cacheItemsMap: {
@@ -13,0 +14,0 @@ [k: string]: DataCache.CacheItem;

dist/fetchData/fetchAllDocuments.js

@@ -8,8 +8,9 @@ import * as tracing_1 from "@effect-ts/core/Tracing";
 import { makeCacheItem } from './makeCacheItem.js';
-export const fetchAllDocuments = ({ databaseTypeDefs, schemaDef, options }) => (T.mapError_(OT.withSpan('@contentlayer/source-notion/fetchData:fetchAllDocuments')(T.map_(T.map_(T.forEachPar_(databaseTypeDefs, (databaseTypeDef) => (T.chain_(S.runCollect(S.chain_(fetchDatabasePages({ databaseTypeDef }), (pages) => (OT.withStreamSpan('@contentlayer/source-notion/fetchData:makeCacheItems')(S.effect(T.forEachParN_(pages, os.cpus().length, (page) => makeCacheItem({
+export const fetchAllDocuments = ({ databaseTypeDefs, previousCache, schemaDef, options }) => (T.mapError_(OT.withSpan('@contentlayer/source-notion/fetchData:fetchAllDocuments')(T.map_(T.map_(T.forEachPar_(databaseTypeDefs, (databaseTypeDef) => (T.chain_(S.runCollect(S.chain_(fetchDatabasePages({ databaseTypeDef }), (pages) => (OT.withStreamSpan('@contentlayer/source-notion/fetchData:makeCacheItems')(S.effect(T.forEachParN_(pages, os.cpus().length, (page) => makeCacheItem({
     page,
     documentTypeDef: schemaDef.documentTypeDefMap[databaseTypeDef.name],
     databaseTypeDef,
+    previousCache,
     options,
-}), fileName_1 + ":25:29")))))), (chunks) => T.reduce_(chunks, [], (z, a) => T.succeed([...z, ...a], fileName_1 + ":38:95"), fileName_1 + ":38:38"), fileName_1 + ":38:16")), fileName_1 + ":19:18"), (chunks) => Chunk.reduce_(chunks, [], (z, a) => [...z, ...a]), fileName_1 + ":41:10"), (documents) => ({ cacheItemsMap: Object.fromEntries(documents.map((_) => [_.document._id, _])) }), fileName_1 + ":42:10")), (error) => new core.SourceFetchDataError({ error, alreadyHandled: false }), fileName_1 + ":44:15"));
+}), fileName_1 + ":26:29")))))), (chunks) => T.reduce_(chunks, [], (z, a) => T.succeed([...z, ...a], fileName_1 + ":40:95"), fileName_1 + ":40:38"), fileName_1 + ":40:16")), fileName_1 + ":20:18"), (chunks) => Chunk.reduce_(chunks, [], (z, a) => [...z, ...a]), fileName_1 + ":43:10"), (documents) => ({ cacheItemsMap: Object.fromEntries(documents.map((_) => [_.document._id, _])) }), fileName_1 + ":44:10")), (error) => new core.SourceFetchDataError({ error, alreadyHandled: false }), fileName_1 + ":46:15"));
 //# sourceMappingURL=fetchAllDocuments.js.map

dist/fetchData/makeCacheItem.d.ts

@@ -9,10 +9,6 @@ import type * as core from '@contentlayer/core';
     page: PageObjectResponse;
+    previousCache: core.DataCache.Cache | undefined;
     options: core.PluginOptions;
 };
-export declare const makeCacheItem: ({ databaseTypeDef, documentTypeDef, page, options }: MakeCacheItemArgs) => T.Effect<import("@contentlayer/utils/effect").Has<import("@notionhq/client").Client> & import("@contentlayer/utils/effect").Has<import("@notion-render/client").NotionRenderer> & import("@contentlayer/utils/effect").Has<OT.Tracer>, unknown, {
-    document: Record<string, any> & core.DocumentMeta;
-    documentHash: string;
-    hasWarnings: boolean;
-    documentTypeName: string;
-}>;
+export declare const makeCacheItem: ({ databaseTypeDef, documentTypeDef, previousCache, page, options }: MakeCacheItemArgs) => T.Effect<import("@contentlayer/utils/effect").Has<import("@notionhq/client").Client> & import("@contentlayer/utils/effect").Has<import("@notion-render/client").NotionRenderer> & import("@contentlayer/utils/effect").Has<OT.Tracer>, unknown, core.DataCache.CacheItem>;
 //# sourceMappingURL=makeCacheItem.d.ts.map

dist/fetchData/makeCacheItem.js

 import * as tracing_1 from "@effect-ts/core/Tracing";
 const fileName_1 = "packages/@contentlayer/source-notion/src/fetchData/makeCacheItem.ts";
-import { hashObject } from '@contentlayer/utils';
 import { OT, pipe, T } from '@contentlayer/utils/effect';
 import { getComputedValues } from './getComputedValues.js';
 import { makeDocument } from './makeDocument.js';
-export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, page, options }) => (OT.withSpan('@contentlayer/source-notion/fetchData:makeCacheItem')(T.chain_(T.gen(function* ($) {
-    const document = yield* $(makeDocument({ documentTypeDef, databaseTypeDef, page, options }), fileName_1 + ":20:32");
-    const computedValues = yield* $(getComputedValues({ document, documentTypeDef }), fileName_1 + ":22:38");
+export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, previousCache, page, options }) => (OT.withSpan('@contentlayer/source-notion/fetchData:makeCacheItem')(T.gen(function* ($) {
+    const documentHash = new Date(page.last_edited_time).getTime().toString();
+    if (previousCache &&
+        previousCache.cacheItemsMap[page.id] &&
+        previousCache.cacheItemsMap[page.id].documentHash === documentHash &&
+        previousCache.cacheItemsMap[page.id].hasWarnings === false) {
+        const cacheItem = previousCache.cacheItemsMap[page.id];
+        return cacheItem;
+    }
+    const document = yield* $(makeDocument({ documentTypeDef, databaseTypeDef, page, options }), fileName_1 + ":33:32");
+    const computedValues = yield* $(getComputedValues({ document, documentTypeDef }), fileName_1 + ":34:38");
     Object.entries(computedValues).forEach(([fieldName, value]) => {
         document[fieldName] = value;
     });
-    return document;
-}, fileName_1 + ":19:10"), (document) => (T.map_(hashObject(document), (hash) => ({
-    document,
-    documentHash: hash,
-    hasWarnings: false,
-    documentTypeName: documentTypeDef.name,
-}), fileName_1 + ":33:14")), fileName_1 + ":30:12")));
+    return {
+        document,
+        documentHash,
+        hasWarnings: false,
+        documentTypeName: documentTypeDef.name,
+    };
+}, fileName_1 + ":20:10")));
 //# sourceMappingURL=makeCacheItem.js.map

dist/index.js

@@ -7,3 +7,3 @@ import * as tracing_1 from "@effect-ts/core/Tracing";
 import * as notion from '@notionhq/client';
-import { fetchAllDocuments } from './fetchData/fetchAllDocuments.js';
+import { fetchData } from './fetchData/fetchData.js';
 import { fetchNotion } from './notion/fetchNotion.js';
@@ -29,3 +29,3 @@ import { provideSchema } from './schema/provideSchema.js';
         provideSchema: () => (T.provideService(NotionRendererTag)(renderer, fileName_1 + ":46:44")(T.provideService(NotionClient)(client, fileName_1 + ":45:39")(provideSchema({ databaseTypeDefs, options })))),
-        fetchData: ({ schemaDef }) => (S.fromEffect((T.provideService(NotionRendererTag)(renderer, fileName_1 + ":59:48")(T.provideService(NotionClient)(client, fileName_1 + ":58:43")(T.either(fetchAllDocuments({
+        fetchData: ({ schemaDef }) => (S.fromEffect((T.provideService(NotionRendererTag)(renderer, fileName_1 + ":59:48")(T.provideService(NotionClient)(client, fileName_1 + ":58:43")(T.either(fetchData({
             databaseTypeDefs: databaseTypeDefs.map((databaseTypeDef) => flattendDatabaseTypeDef(databaseTypeDef)),
@@ -32,0 +32,0 @@ schemaDef,

dist/notion/fetchNotion.js

 import * as tracing_1 from "@effect-ts/core/Tracing";
 const fileName_1 = "packages/@contentlayer/source-notion/src/notion/fetchNotion.ts";
-import PQueue from 'p-queue';
+import PQueue, { AbortError } from 'p-queue';
 import pRetry from 'p-retry';
 const queue = new PQueue({ interval: 1000, intervalCap: 4 });
 export const fetchNotion = (...args) => {
-    return queue.add(() => pRetry(() => fetch(...args), {
+    return queue.add(() => pRetry(async () => {
+        const response = await fetch(...args);
+        if (!response.ok) {
+            throw new AbortError(response.statusText);
+        }
+        return response;
+    }, {
         retries: 5,
@@ -9,0 +15,0 @@ onFailedAttempt: async () => new Promise((res) => setTimeout(res, 1000)),

package.json

 {
   "name": "contentlayer-source-notion",
-  "version": "0.0.1-alpha.16",
+  "version": "0.0.1-alpha.17",
   "type": "module",
@@ -5,0 +5,0 @@ "exports": "./dist/index.js",

src/fetchData/fetchAllDocuments.ts

@@ -13,2 +13,3 @@ import * as os from 'node:os'
   databaseTypeDefs: DatabaseTypeDef[]
+  previousCache: core.DataCache.Cache | undefined
   schemaDef: core.SchemaDef
@@ -18,3 +19,3 @@ options: core.PluginOptions
 
-export const fetchAllDocuments = ({ databaseTypeDefs, schemaDef, options }: FetchAllDocumentsArgs) =>
+export const fetchAllDocuments = ({ databaseTypeDefs, previousCache, schemaDef, options }: FetchAllDocumentsArgs) =>
   pipe(
@@ -32,2 +33,3 @@ T.forEachPar_(databaseTypeDefs, (databaseTypeDef) =>
                   databaseTypeDef,
+                  previousCache,
                   options,
@@ -34,0 +36,0 @@ }),

src/fetchData/makeCacheItem.ts

@@ -14,10 +14,22 @@ import type * as core from '@contentlayer/core'
   page: PageObjectResponse
+  previousCache: core.DataCache.Cache | undefined
   options: core.PluginOptions
 }
 
-export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, page, options }: MakeCacheItemArgs) =>
+export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, previousCache, page, options }: MakeCacheItemArgs) =>
   pipe(
     T.gen(function* ($) {
+      const documentHash = new Date(page.last_edited_time).getTime().toString()
+
+      if (
+        previousCache &&
+        previousCache.cacheItemsMap[page.id] &&
+        previousCache.cacheItemsMap[page.id]!.documentHash === documentHash &&
+        previousCache.cacheItemsMap[page.id]!.hasWarnings === false
+      ) {
+        const cacheItem = previousCache.cacheItemsMap[page.id]!
+        return cacheItem
+      }
+
       const document = yield* $(makeDocument({ documentTypeDef, databaseTypeDef, page, options }))
-
       const computedValues = yield* $(getComputedValues({ document, documentTypeDef }))
@@ -29,16 +41,10 @@
 
-      return document
+      return {
+        document,
+        documentHash,
+        hasWarnings: false,
+        documentTypeName: documentTypeDef.name,
+      }
     }),
-    T.chain((document) =>
-      pipe(
-        hashObject(document),
-        T.map((hash) => ({
-          document,
-          documentHash: hash,
-          hasWarnings: false,
-          documentTypeName: documentTypeDef.name,
-        })),
-      ),
-    ),
     OT.withSpan('@contentlayer/source-notion/fetchData:makeCacheItem'),
   )

src/index.ts

@@ -7,3 +7,3 @@ import type * as core from '@contentlayer/core'
 
-import { fetchAllDocuments } from './fetchData/fetchAllDocuments.js'
+import { fetchData } from './fetchData/fetchData.js'
 import { fetchNotion } from './notion/fetchNotion.js'
@@ -53,3 +53,3 @@ import { provideSchema } from './schema/provideSchema.js'
           pipe(
-            fetchAllDocuments({
+            fetchData({
               databaseTypeDefs: databaseTypeDefs.map((databaseTypeDef) => flattendDatabaseTypeDef(databaseTypeDef)),
@@ -56,0 +56,0 @@ schemaDef,

src/notion/fetchNotion.ts

@@ -1,2 +0,2 @@
-import PQueue from 'p-queue'
+import PQueue, { AbortError } from 'p-queue'
 import pRetry from 'p-retry'
@@ -8,7 +8,18 @@
   return queue.add(() =>
-    pRetry(() => fetch(...args), {
-      retries: 5,
-      onFailedAttempt: async () => new Promise((res) => setTimeout(res, 1000)),
-    }),
+    pRetry(
+      async () => {
+        const response = await fetch(...args)
+
+        if (!response.ok) {
+          throw new AbortError(response.statusText)
+        }
+
+        return response
+      },
+      {
+        retries: 5,
+        onFailedAttempt: async () => new Promise((res) => setTimeout(res, 1000)),
+      },
+    ),
   ) as Promise<Response>
 }

No alert changes

Improved metrics

Total package byte prevSize

170980

increased by3.73%

Number of package files

217

increased by2.36%

Lines of code

2320

increased by3.39%

No dependency changes