@enterprise_search/indexing - npm Package Compare versions

Comparing version 0.7.56 to 0.7.57

dist/src/access.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.access = exports.defaultIndexingContext = exports.callFetch = exports.toUrlFn = exports.optionsFrom = exports.headersFrom = exports.findVarsFrom = exports.defaultAuthFn = exports.getOrUpdateEntraId = exports.authForEntraId = void 0;
+exports.findVarsFrom = exports.defaultAuthFn = exports.authForEntraId = void 0;
+exports.getOrUpdateEntraId = getOrUpdateEntraId;
+exports.headersFrom = headersFrom;
+exports.optionsFrom = optionsFrom;
+exports.toUrlFn = toUrlFn;
+exports.callFetch = callFetch;
+exports.defaultIndexingContext = defaultIndexingContext;
+exports.access = access;
 const tree_index_1 = require("./tree.index");
@@ -97,3 +104,2 @@ const forest_index_1 = require("./forest.index");
 }
-exports.getOrUpdateEntraId = getOrUpdateEntraId;
 const globalTokenCache = {};
@@ -137,3 +143,2 @@ const defaultAuthFn = (env, fetch, timeService, tokenCache = globalTokenCache) => async (auth) => {
 }
-exports.headersFrom = headersFrom;
 async function optionsFrom(o) {
@@ -145,3 +150,2 @@ if (!o)
 }
-exports.optionsFrom = optionsFrom;
 function toUrlFn(fetch, optionsFn) {
@@ -165,3 +169,2 @@ return async (url) => {
 }
-exports.toUrlFn = toUrlFn;
 async function callFetch(fetchFn, url, options, prefix) {
@@ -176,3 +179,2 @@ const resp = await fetchFn(url, options);
 }
-exports.callFetch = callFetch;
 function defaultIndexingContext(env, fetch, metricsConfig, metrics) {
@@ -209,3 +211,2 @@ return {
 }
-exports.defaultIndexingContext = defaultIndexingContext;
 async function access(ic, details, offsetUrl, config, resfn) {
@@ -236,2 +237,1 @@ const { method, body, extraHeaders, pagingFn } = config;
 }
-exports.access = access;

dist/src/column.details.js

@@ -26,3 +26,6 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toColumnDetails = exports.valueFromColumnDetails = exports.removeHtmlTags = exports.toJsonFromNameAndValue = void 0;
+exports.toColumnDetails = void 0;
+exports.toJsonFromNameAndValue = toJsonFromNameAndValue;
+exports.removeHtmlTags = removeHtmlTags;
+exports.valueFromColumnDetails = valueFromColumnDetails;
 const cheerio = __importStar(require("cheerio"));
@@ -34,3 +37,2 @@ function toJsonFromNameAndValue(nameAndValue) {
 }
-exports.toJsonFromNameAndValue = toJsonFromNameAndValue;
 function removeHtmlTags(text) {
@@ -45,3 +47,2 @@ try {
 }
-exports.removeHtmlTags = removeHtmlTags;
 function valueFromColumnDetails(details, value) {
@@ -56,3 +57,2 @@ if (details.value)
 }
-exports.valueFromColumnDetails = valueFromColumnDetails;
 function normaliseColumnDetails(raw) {
@@ -59,0 +59,0 @@ return {

dist/src/fetch.log.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.withFetchLog = void 0;
+exports.withFetchLog = withFetchLog;
 const access_1 = require("./access");
@@ -23,2 +23,1 @@ const kleislis_1 = require("@enterprise_search/kleislis");
 }
-exports.withFetchLog = withFetchLog;

dist/src/forest.index.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.indexForestOfTrees = exports.rememberForestLogsAndMetrics = exports.defaultForestLogAndMetrics = exports.nullIndexForestLogAndMetrics = exports.consoleIndexForestLogAndMetrics = exports.metricsIndexMetrics = exports.consoleIndexMetrics = exports.addNonFunctionalsToIndexForestTc = void 0;
+exports.nullIndexForestLogAndMetrics = exports.consoleIndexForestLogAndMetrics = exports.metricsIndexMetrics = exports.consoleIndexMetrics = void 0;
+exports.addNonFunctionalsToIndexForestTc = addNonFunctionalsToIndexForestTc;
+exports.defaultForestLogAndMetrics = defaultForestLogAndMetrics;
+exports.rememberForestLogsAndMetrics = rememberForestLogsAndMetrics;
+exports.indexForestOfTrees = indexForestOfTrees;
 const utils_1 = require("@laoban/utils");
@@ -13,3 +17,2 @@ const kleislis_1 = require("@enterprise_search/kleislis");
 }
-exports.addNonFunctionalsToIndexForestTc = addNonFunctionalsToIndexForestTc;
 exports.consoleIndexMetrics = {
@@ -68,3 +71,2 @@ start: (name) => console.log('start', name),
 }
-exports.defaultForestLogAndMetrics = defaultForestLogAndMetrics;
 function rememberForestLogsAndMetrics(msgs) {
@@ -78,3 +80,2 @@ return {
 }
-exports.rememberForestLogsAndMetrics = rememberForestLogsAndMetrics;
 function indexForestOfTrees(logAndMetrics, tc, paging, //the string is the treeId
@@ -106,3 +107,2 @@ processTreeRoot) {
 }
-exports.indexForestOfTrees = indexForestOfTrees;
 ;

dist/src/index.into.file.js

@@ -6,3 +6,7 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.appendFile = exports.addToFile = exports.insertIntoFileWithNonFunctionals = exports.addFileToQueue = exports.waitUntilQueueEmpty = exports.insertIntoFile = void 0;
+exports.addToFile = exports.insertIntoFileWithNonFunctionals = void 0;
+exports.insertIntoFile = insertIntoFile;
+exports.waitUntilQueueEmpty = waitUntilQueueEmpty;
+exports.addFileToQueue = addFileToQueue;
+exports.appendFile = appendFile;
 const promises_1 = __importDefault(require("fs/promises"));
@@ -43,3 +47,2 @@ const utils_1 = require("@laoban/utils");
 }
-exports.insertIntoFile = insertIntoFile;
 const queue = [];
@@ -61,3 +64,2 @@ function waitUntilQueueEmpty(inProgressMsg, finishedMsg, interval) {
 }
-exports.waitUntilQueueEmpty = waitUntilQueueEmpty;
 const appendWithRetry = (0, kleislis_1.withRetry)(kleislis_1.defaultRetryPolicy, appendFile);
@@ -88,3 +90,2 @@ let processing = false;
 }
-exports.addFileToQueue = addFileToQueue;
 const insertIntoFileWithNonFunctionals = (rootDir, fileTemplate, _index, nf, indexMetrics) => (0, tree_index_1.addNonFunctionsToIndexer)(nf, insertIntoFile({
@@ -120,2 +121,1 @@ file: (id) => {
 }
-exports.appendFile = appendFile;

dist/src/index.parent.child.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.indexParentChild = exports.addNonFunctionalsToIndexParentChildTc = exports.rememberIndexParentChildLogsAndMetrics = exports.defaultIndexParentChildLogAndMetrics = exports.consoleIndexParentChildLogAndMetrics = exports.nullIndexParentChildLogAndMetrics = void 0;
+exports.rememberIndexParentChildLogsAndMetrics = exports.consoleIndexParentChildLogAndMetrics = exports.nullIndexParentChildLogAndMetrics = void 0;
+exports.defaultIndexParentChildLogAndMetrics = defaultIndexParentChildLogAndMetrics;
+exports.addNonFunctionalsToIndexParentChildTc = addNonFunctionalsToIndexParentChildTc;
+exports.indexParentChild = indexParentChild;
 const utils_1 = require("@laoban/utils");
@@ -45,3 +48,2 @@ const kleislis_1 = require("@enterprise_search/kleislis");
 }
-exports.defaultIndexParentChildLogAndMetrics = defaultIndexParentChildLogAndMetrics;
 const rememberIndexParentChildLogsAndMetrics = (msgs) => {
@@ -63,3 +65,2 @@ return {
 }
-exports.addNonFunctionalsToIndexParentChildTc = addNonFunctionalsToIndexParentChildTc;
 function indexParentChild(logAndMetrics, tc, pc, transformer, childId, executeOptions) {
@@ -97,2 +98,1 @@ return (indexer) => async (parentId) => {
 }
-exports.indexParentChild = indexParentChild;

dist/src/indexer.domain.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.rememberIndex = exports.consoleIndexer = exports.indexerWithTransformer = void 0;
+exports.consoleIndexer = void 0;
+exports.indexerWithTransformer = indexerWithTransformer;
+exports.rememberIndex = rememberIndex;
 function indexerWithTransformer(indexer, fn) {
@@ -12,3 +14,2 @@ return {
 }
-exports.indexerWithTransformer = indexerWithTransformer;
 exports.consoleIndexer = {
@@ -28,2 +29,1 @@ start: async (rootId) => { console.log(`Started: ${rootId}`); },
 }
-exports.rememberIndex = rememberIndex;

dist/src/indexing.fixture.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.indexParentChildForTestTc = exports.treeIndexForTestTc = exports.indexForestTc = exports.forestAndPageToResult = exports.getFromTestForest = exports.TestPageQueryPC = exports.indexTestFolder = void 0;
+exports.indexParentChildForTestTc = exports.treeIndexForTestTc = exports.indexForestTc = exports.TestPageQueryPC = exports.indexTestFolder = void 0;
+exports.getFromTestForest = getFromTestForest;
+exports.forestAndPageToResult = forestAndPageToResult;
 const utils_1 = require("@laoban/utils");
@@ -44,3 +46,2 @@ exports.indexTestFolder = {
 }
-exports.getFromTestForest = getFromTestForest;
 function forestAndPageToResult(forest, page) {
@@ -57,3 +58,2 @@ const all = [...(forest.leafs), ...(Object.entries(forest.folders || {}))];
 }
-exports.forestAndPageToResult = forestAndPageToResult;
 exports.indexForestTc = {
@@ -60,0 +60,0 @@ fetchForest: async (forestId, page) => {

dist/src/indexing.non.functionals.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.defaultIndexTreeNfs = exports.treeRetryPolicy = exports.indexThrottlePrototype = exports.queryThrottlePrototype = exports.treeNonFunctionals = exports.treeNonFunctionalsFromConfig = void 0;
+exports.treeRetryPolicy = exports.indexThrottlePrototype = exports.queryThrottlePrototype = void 0;
+exports.treeNonFunctionalsFromConfig = treeNonFunctionalsFromConfig;
+exports.treeNonFunctionals = treeNonFunctionals;
+exports.defaultIndexTreeNfs = defaultIndexTreeNfs;
 const kleislis_1 = require("@enterprise_search/kleislis");
@@ -21,3 +24,2 @@ function treeNonFunctionalsFromConfig(indexName, indexItem) {
 }
-exports.treeNonFunctionalsFromConfig = treeNonFunctionalsFromConfig;
 function treeNonFunctionals(indexName, queryThrottle, retryPolicy, indexThrottle, concurrencyLimit = 1000, indexerConcurrencyLimit = 2) {
@@ -39,3 +41,2 @@ return {
 }
-exports.treeNonFunctionals = treeNonFunctionals;
 //mostly for testing, or if we don't have a config file
@@ -47,3 +48,3 @@ exports.queryThrottlePrototype = { current: 0, max: 50, throttlingDelay: 50, tokensPer100ms: 0.3 }; //maybe a bit fast. per hour this is 12000. That's my personal cap I think
     maximumInterval: 30000,
-    maximumAttempts: 50,
+    maximumAttempts: 50, //we do want to get there. This is a seriously slow retry. We have a decent throttling policy.I think this is OK and not abusive
     nonRecoverableErrors: ['Not Found']
@@ -54,2 +55,1 @@ };
 }
-exports.defaultIndexTreeNfs = defaultIndexTreeNfs;

dist/src/paging.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchLeavesForTree = exports.fetchArrayWithPaging = exports.fetchWithPaging = exports.fetchOneItem = exports.mapWithPaging = exports.NoPagingTc = exports.noPagingAccessConfig = void 0;
+exports.NoPagingTc = exports.noPagingAccessConfig = void 0;
+exports.mapWithPaging = mapWithPaging;
+exports.fetchOneItem = fetchOneItem;
+exports.fetchWithPaging = fetchWithPaging;
+exports.fetchArrayWithPaging = fetchArrayWithPaging;
+exports.fetchLeavesForTree = fetchLeavesForTree;
 exports.noPagingAccessConfig = {
@@ -17,3 +22,2 @@ pagingFn: (json, linkHeader) => ({})
 }
-exports.mapWithPaging = mapWithPaging;
 function fetchOneItem(fetch) {
@@ -33,3 +37,2 @@ return async (url, options, respFn) => {
 }
-exports.fetchOneItem = fetchOneItem;
 const fetchOneForPaging = (fetchFn, tc, log) => async (url, options, paging) => {
@@ -73,3 +76,2 @@ const fullUrl = tc.url(url, paging);
 }
-exports.fetchWithPaging = fetchWithPaging;
 function fetchArrayWithPaging(fetch, log, tc) {
@@ -91,3 +93,2 @@ const f = fetchWithPaging(fetch, log, tc);
 }
-exports.fetchArrayWithPaging = fetchArrayWithPaging;
 //Note we could easily refactor this and make it shorter. But when trying to understand it, and single step through it, I find it easier like this.
@@ -127,2 +128,1 @@ //An interesting case where I think cohession wins out over DRY
 }
-exports.fetchLeavesForTree = fetchLeavesForTree;

dist/src/tree.index.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.processTreeRoot = exports.rememberIndexTreeLogAndMetrics = exports.defaultTreeLogAndMetrics = exports.consoleIndexTreeLogAndMetrics = exports.nullIndexTreeLogAndMetrics = exports.stopNonFunctionals = exports.addNonFunctionsToIndexer = exports.addNonFunctionalsToIndexTreeTC = void 0;
+exports.consoleIndexTreeLogAndMetrics = exports.nullIndexTreeLogAndMetrics = void 0;
+exports.addNonFunctionalsToIndexTreeTC = addNonFunctionalsToIndexTreeTC;
+exports.addNonFunctionsToIndexer = addNonFunctionsToIndexer;
+exports.stopNonFunctionals = stopNonFunctionals;
+exports.defaultTreeLogAndMetrics = defaultTreeLogAndMetrics;
+exports.rememberIndexTreeLogAndMetrics = rememberIndexTreeLogAndMetrics;
+exports.processTreeRoot = processTreeRoot;
 const kleislis_1 = require("@enterprise_search/kleislis");
@@ -15,3 +21,2 @@ const utils_1 = require("@laoban/utils");
 }
-exports.addNonFunctionalsToIndexTreeTC = addNonFunctionalsToIndexTreeTC;
 function addNonFunctionsToIndexer(nf, indexer) {
@@ -25,3 +30,2 @@ return {
 }
-exports.addNonFunctionsToIndexer = addNonFunctionsToIndexer;
 function stopNonFunctionals(nft) {
@@ -31,3 +35,2 @@ (0, kleislis_1.stopThrottling)(nft.query.throttle);
 }
-exports.stopNonFunctionals = stopNonFunctionals;
 exports.nullIndexTreeLogAndMetrics = {
@@ -82,3 +85,2 @@ leafIds: () => { },
 }
-exports.defaultTreeLogAndMetrics = defaultTreeLogAndMetrics;
 function rememberIndexTreeLogAndMetrics(msgs) {
@@ -94,3 +96,2 @@ return {
 }
-exports.rememberIndexTreeLogAndMetrics = rememberIndexTreeLogAndMetrics;
 function processTreeRoot(logAndMetrics, tc, pc, indexer, options) {
@@ -149,2 +150,1 @@ async function processFolder(rootId, folderId, parentId) {
 }
-exports.processTreeRoot = processTreeRoot;

package.json

 {
   "name": "@enterprise_search/indexing",
   "description": "general code to help with indexing",
-  "version": "0.7.56",
+  "version": "0.7.57",
   "main": "dist/index",
@@ -27,4 +27,4 @@ "types": "dist/index",
     "@itsmworkbench/utils": "0.3.12",
-    "@enterprise_search/indexconfig": "0.7.56",
-    "@enterprise_search/kleislis": "0.7.56",
+    "@enterprise_search/indexconfig": "0.7.57",
+    "@enterprise_search/kleislis": "0.7.57",
     "debug": "^4.3.4"
@@ -31,0 +31,0 @@ },

No alert changes

Worsened metrics

Total package byte prevSize

89325

decreased by-1.19%

Lines of code

1931

decreased by-0.16%

Dependency changes

• + Added@enterprise_search/indexconfig@0.7.57(transitive)

• + Added@enterprise_search/kleislis@0.7.57(transitive)

• + Added@enterprise_search/tika@0.7.57(transitive)

• - Removed@enterprise_search/indexconfig@0.7.56(transitive)

• - Removed@enterprise_search/kleislis@0.7.56(transitive)

• - Removed@enterprise_search/tika@0.7.56(transitive)

• Updated@enterprise_search/indexconfig@0.7.57

• Updated@enterprise_search/kleislis@0.7.57