@enterprise_search/indexing
Advanced tools
@enterprise_search/indexing - npm Package Compare versions
Comparing version 0.8.0 to 0.8.1
@@ -10,3 +10,3 @@ import { NameAnd } from "@laoban/utils"; | ||
| import { AiClientMaker } from "@enterprise_search/clients"; | ||
| import { Digest } from "@enterprise_search/fileutils"; | ||
| import { Digest } from "@enterprise_search/utils"; | ||
| export type SourceSinkDetails = { | ||
@@ -75,3 +75,3 @@ baseurl: string; | ||
| }; | ||
| export declare function defaultIndexingContext(env: NameAnd<string>, fetch: FetchFn, metricsConfig: MetricsConfig, metrics: NameAnd<number>): IndexingContext; | ||
| export declare function defaultIndexingContext(env: NameAnd<string>, fetch: FetchFn, metricsConfig: MetricsConfig, metrics: NameAnd<number>, digest: Digest): IndexingContext; | ||
| export type AccessConfig<L> = { | ||
@@ -78,0 +78,0 @@ method?: 'Get' | 'Post' | 'Put' | 'Delete'; |
@@ -19,3 +19,2 @@ "use strict"; | ||
| const openai_1 = require("@enterprise_search/openai"); | ||
| const fileutils_1 = require("@enterprise_search/fileutils"); | ||
| function authForApiToken(env, auth) { | ||
@@ -200,5 +199,5 @@ const apiKey = auth.credentials?.apiKey; | ||
| } | ||
| function defaultIndexingContext(env, fetch, metricsConfig, metrics) { | ||
| function defaultIndexingContext(env, fetch, metricsConfig, metrics, digest) { | ||
| return { | ||
| digest: fileutils_1.calculateSHA256, | ||
| digest, | ||
| aiClientMaker: (0, openai_1.defaultOpenAiMaker)(env, fetch), | ||
@@ -205,0 +204,0 @@ env, |
@@ -0,1 +1,2 @@ | ||
| import { RemoveHtmlTags } from "@enterprise_search/utils"; | ||
| export type NameAndValueColumn = { | ||
@@ -14,4 +15,3 @@ name: string; | ||
| export declare function toJsonFromNameAndValue(nameAndValue?: NameAndValueColumn[]): Record<string, string>; | ||
| export declare function removeHtmlTags(text: string): string; | ||
| export declare function valueFromColumnDetails(details: ColumnDetails, value?: string): any; | ||
| export declare const valueFromColumnDetails: (removeHtmlTags: RemoveHtmlTags) => (details: ColumnDetails, value?: string) => any; | ||
| export declare const toColumnDetails: (raw: (string | ColumnDetails)[]) => ColumnDetails[]; |
| "use strict"; | ||
| var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { | ||
| if (k2 === undefined) k2 = k; | ||
| var desc = Object.getOwnPropertyDescriptor(m, k); | ||
| if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { | ||
| desc = { enumerable: true, get: function() { return m[k]; } }; | ||
| } | ||
| Object.defineProperty(o, k2, desc); | ||
| }) : (function(o, m, k, k2) { | ||
| if (k2 === undefined) k2 = k; | ||
| o[k2] = m[k]; | ||
| })); | ||
| var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { | ||
| Object.defineProperty(o, "default", { enumerable: true, value: v }); | ||
| }) : function(o, v) { | ||
| o["default"] = v; | ||
| }); | ||
| var __importStar = (this && this.__importStar) || function (mod) { | ||
| if (mod && mod.__esModule) return mod; | ||
| var result = {}; | ||
| if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); | ||
| __setModuleDefault(result, mod); | ||
| return result; | ||
| }; | ||
| Object.defineProperty(exports, "__esModule", { value: true }); | ||
| exports.toColumnDetails = void 0; | ||
| exports.toColumnDetails = exports.valueFromColumnDetails = void 0; | ||
| exports.toJsonFromNameAndValue = toJsonFromNameAndValue; | ||
| exports.removeHtmlTags = removeHtmlTags; | ||
| exports.valueFromColumnDetails = valueFromColumnDetails; | ||
| const cheerio = __importStar(require("cheerio")); | ||
| function toJsonFromNameAndValue(nameAndValue) { | ||
@@ -36,12 +10,3 @@ if (!nameAndValue) | ||
| } | ||
| function removeHtmlTags(text) { | ||
| try { | ||
| let $ = cheerio.load(`<body>${text}</body>`); | ||
| return $("body").text(); | ||
| } | ||
| catch (e) { | ||
| return text; | ||
| } | ||
| } | ||
| function valueFromColumnDetails(details, value) { | ||
| const valueFromColumnDetails = (removeHtmlTags) => (details, value) => { | ||
| if (details.value) | ||
@@ -54,3 +19,4 @@ return details.value; | ||
| return value; | ||
| } | ||
| }; | ||
| exports.valueFromColumnDetails = valueFromColumnDetails; | ||
| function normaliseColumnDetails(raw) { | ||
@@ -57,0 +23,0 @@ return { |
| import { Timeservice } from "@itsmworkbench/utils"; | ||
| import { FetchFn } from "./access"; | ||
| export declare function withFetchLog(name: string, timeService: Timeservice, fn: FetchFn): FetchFn; | ||
| import { SimpleLogFn } from "@enterprise_search/kleislis"; | ||
| export declare function withFetchLog(timeService: Timeservice, log: SimpleLogFn, fn: FetchFn): FetchFn; |
@@ -6,4 +6,3 @@ "use strict"; | ||
| const kleislis_1 = require("@enterprise_search/kleislis"); | ||
| function withFetchLog(name, timeService, fn) { | ||
| const log = (0, kleislis_1.nameLogFn)(name); | ||
| function withFetchLog(timeService, log, fn) { | ||
| return async (url, options) => { | ||
@@ -10,0 +9,0 @@ const realOptions = await (0, access_1.optionsFrom)(options); |
| { | ||
| "name": "@enterprise_search/indexing", | ||
| "description": "general code to help with indexing", | ||
| "version": "0.8.0", | ||
| "version": "0.8.1", | ||
| "main": "dist/index", | ||
@@ -23,8 +23,7 @@ "types": "dist/index", | ||
| "dependencies": { | ||
| "cheerio": "^1.0.0-rc.12", | ||
| "@laoban/utils": "1.4.51", | ||
| "@laoban/variables": "1.4.51", | ||
| "@itsmworkbench/utils": "0.3.12", | ||
| "@enterprise_search/indexconfig": "0.8.0", | ||
| "@enterprise_search/kleislis": "0.8.0", | ||
| "@enterprise_search/indexconfig": "0.8.1", | ||
| "@enterprise_search/kleislis": "0.8.1", | ||
| "debug": "^4.3.4" | ||
@@ -31,0 +30,0 @@ }, |
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Improved metrics
- Dependency count
- decreased by-14.29%
6
Worsened metrics
- Total package byte prevSize
- decreased by-1.02%
154354
- Lines of code
- decreased by-1.77%
1945
Dependency changes
+ Added@enterprise_search/indexconfig@0.8.1(transitive)
+ Added@enterprise_search/kleislis@0.8.1(transitive)
+ Added@enterprise_search/tika@0.8.1(transitive)
- Removedcheerio@^1.0.0-rc.12
- Removed@enterprise_search/indexconfig@0.8.0(transitive)
- Removed@enterprise_search/kleislis@0.8.0(transitive)
- Removed@enterprise_search/tika@0.8.0(transitive)
- Removedboolbase@1.0.0(transitive)
- Removedcheerio@1.0.0(transitive)
- Removedcheerio-select@2.1.0(transitive)
- Removedcss-select@5.1.0(transitive)
- Removedcss-what@6.1.0(transitive)
- Removeddom-serializer@2.0.0(transitive)
- Removeddomelementtype@2.3.0(transitive)
- Removeddomhandler@5.0.3(transitive)
- Removeddomutils@3.2.2(transitive)
- Removedencoding-sniffer@0.2.0(transitive)
- Removedentities@4.5.0(transitive)
- Removedhtmlparser2@9.1.0(transitive)
- Removediconv-lite@0.6.3(transitive)
- Removednth-check@2.1.1(transitive)
- Removedparse5@7.2.1(transitive)
- Removedparse5-htmlparser2-tree-adapter@7.1.0(transitive)
- Removedparse5-parser-stream@7.1.2(transitive)
- Removedsafer-buffer@2.1.2(transitive)
- Removedundici@6.21.1(transitive)
- Removedwhatwg-encoding@3.1.1(transitive)
- Removedwhatwg-mimetype@4.0.0(transitive)