pagefind - npm Package Compare versions

Comparing version 1.0.0-alpha.5 to 1.0.0-alpha.6

lib/index.js

@@ -40,9 +40,22 @@ import { PagefindService } from "./service.js";
  * 
+ * @param {import('pagefindService').PagefindServiceConfig=} config
  * @type {import('pagefindService').createIndex} 
  * */
-export const createIndex = () => new Promise((resolve, reject) => {
+export const createIndex = (config) => new Promise((resolve, reject) => {
+
+    // TODO: Validate `config` here, instead of waiting for the backend to throw an error.
+    // Ideally we create a global Pagefind config JSON schema that (a subset of) can be used here.
+
     const action = 'NewIndex';
     launch().sendMessage(
         {
-            type: action
+            type: action,
+            config: {
+                root_selector: config?.rootSelector,
+                exclude_selectors: config?.excludeSelectors,
+                force_language: config?.forceLanguage,
+                verbose: config?.verbose,
+                logfile: config?.logfile,
+                keep_index_url: config?.keepIndexUrl,
+            }
         }, (response) => {
@@ -73,4 +86,6 @@ /** @type {function(InternalResponsePayload): Omit<NewIndexResponse, 'errors'>?} */
         addCustomRecord: (record) => addCustomRecord(indexId, record),
-        writeFiles: () => writeFiles(indexId),
-        getFiles: () => getFiles(indexId)
+        addDirectory: (dir) => addDirectory(indexId, dir),
+        writeFiles: (options) => writeFiles(indexId, options),
+        getFiles: () => getFiles(indexId),
+        deleteIndex: () => deleteIndex(indexId)
     }
@@ -155,9 +170,44 @@ }
 
+
 /**
+ * @typedef {import('pagefindService').IndexingResponse} IndexingResponse
+ * 
+ * @param {number} indexId 
+ * @param {import('pagefindService').SiteDirectory} dir
+ * @returns {Promise<IndexingResponse>}
+ */
+const addDirectory = (indexId, dir) => new Promise((resolve, reject) => {
+    const action = 'AddDir';
+    const responseAction = 'IndexedDir';
+    launch().sendMessage(
+        {
+            type: action,
+            index_id: indexId,
+            path: dir.path,
+            glob: dir.glob
+        }, (response) => {
+            /** @type {function(InternalResponsePayload): Omit<IndexingResponse, 'errors'>?} */
+            const successCallback = (success) => {
+                if (success.type !== responseAction) {
+                    reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
+                    return null;
+                }
+
+                return {
+                    page_count: success.page_count
+                }
+            };
+            handleApiResponse(resolve, reject, response, successCallback);
+        }
+    );
+});
+
+/**
  * @typedef {import ('pagefindService').WriteFilesResponse} WriteFilesResponse
  * 
  * @param {number} indexId 
+ * @param {import('pagefindService').WriteOptions=} options
  * @returns {Promise<WriteFilesResponse>}
  */
-const writeFiles = (indexId) => new Promise((resolve, reject) => {
+const writeFiles = (indexId, options) => new Promise((resolve, reject) => {
     const action = 'WriteFiles';
@@ -168,2 +218,3 @@ launch().sendMessage(
             index_id: indexId,
+            bundle_path: options?.bundlePath
         }, (response) => {
@@ -178,3 +229,3 @@ /** @type {function(InternalResponsePayload): Omit<WriteFilesResponse, 'errors'>?} */
                 return {
-                    bundleLocation: success.bundle_location
+                    bundlePath: success.bundle_path
                 }
@@ -219,2 +270,27 @@ };
     );
-});
+});
+
+/**
+ * @param {number} indexId 
+ * @returns {Promise<null>}
+ */
+const deleteIndex = (indexId) => new Promise((resolve, reject) => {
+    const action = 'DeleteIndex';
+    launch().sendMessage(
+        {
+            type: action,
+            index_id: indexId,
+        }, (response) => {
+            /** @type {function(InternalResponsePayload): Omit<GetFilesResponse, 'errors'>?} */
+            const successCallback = (success) => {
+                if (success.type !== action) {
+                    reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
+                    return null;
+                }
+
+                return null;
+            };
+            handleApiResponse(resolve, reject, response, successCallback);
+        }
+    );
+});

lib/service.js

@@ -98,2 +98,25 @@ import child_process from 'child_process';
         let parsed_message = PagefindService.parseMessage(message);
+        if (parsed_message && typeof parsed_message.message_id !== "number") {
+            if (parsed_message.payload.type !== "Error") {
+                // Unreachable (hopefully)
+                return;
+            }
+            if (!parsed_message.payload.original_message) {
+                throw new Error(`Failed to communicate with the Pagefind service backend: ${parsed_message.payload.message}`);
+            }
+            try {
+                let our_message = JSON.parse(parsed_message.payload.original_message);
+                if (our_message.message_id && this.callbacks[our_message.message_id]) {
+                    this.returnValue(
+                        our_message.message_id,
+                        {
+                            exception: new Error(`Pagefind service error when parsing a message: ${parsed_message.payload.message}\nMessage being parsed:\n${parsed_message.payload.original_message}`),
+                            err: null,
+                            result: null,
+                        });
+                }
+            } catch (e) {
+                throw new Error(`Failed to communicate with the Pagefind service backend: ${parsed_message.payload.message}`);
+            }
+        }
         if (parsed_message && this.callbacks[parsed_message.message_id]) {
@@ -131,4 +154,3 @@ const isError = parsed_message.payload.type === "Error";
         if (this.backend === null) {
-            console.error(`Cannot send message, backend is closed: `, message);
-            return;
+            throw new Error(`Cannot send message, backend is closed: ${message}`);
         }
@@ -176,6 +198,5 @@ let wrapped_message = this.wrapOutgoingMessage(message, callback);
         } catch {
-            console.error("Failed to parse message from backend");
-            return null;
+            throw new Error(`Failed to parse a message from the Pagefind service backend`);
         }
     }
 }

package.json

 {
   "name": "pagefind",
-  "version": "1.0.0-alpha.5",
+  "version": "1.0.0-alpha.6",
   "type": "module",
@@ -22,7 +22,7 @@ "description": "Implement search on any static website.",
   "optionalDependencies": {
-    "@pagefind/linux-x64": "1.0.0-alpha.5",
-    "@pagefind/linux-arm64": "1.0.0-alpha.5",
-    "@pagefind/darwin-x64": "1.0.0-alpha.5",
-    "@pagefind/darwin-arm64": "1.0.0-alpha.5",
-    "@pagefind/windows-x64": "1.0.0-alpha.5"
+    "@pagefind/linux-x64": "1.0.0-alpha.6",
+    "@pagefind/linux-arm64": "1.0.0-alpha.6",
+    "@pagefind/darwin-x64": "1.0.0-alpha.6",
+    "@pagefind/darwin-arm64": "1.0.0-alpha.6",
+    "@pagefind/windows-x64": "1.0.0-alpha.6"
   },
@@ -29,0 +29,0 @@ "keywords": [

README.md

@@ -1,3 +0,196 @@
-# Pagefind
+# Pagefind Static Search
 
-The CLI for Pagefind.
+Pagefind is a fully static search library that aims to perform well on large sites, while using as little of your users’ bandwidth as possible, and without hosting any infrastructure.
+
+The full documentation on using Pagefind can be found at https://pagefind.app/.
+
+This packages houses a wrapper for running the precompiled Pagefind binary, and also serves as a NodeJS indexing library that can be integrated into existing tools.
+
+## Running Pagefind through NPX
+
+This is the recommended way of running Pagefind on a static site.
+
+```
+npx pagefind --source "public"
+```
+
+For more details on using the Pagefind binary, see [Installing and running Pagefind](https://pagefind.app/docs/installation/#running-via-npx), and the rest of the Pagefind documentation.
+
+## Using Pagefind as a Node library
+
+This package also provides an interface to the Pagefind binary directly as a package you can import.
+This generally isn't required, and running the binary directly on your source code is the recommended approach
+for the majority of use-cases.
+
+The rest of this documentation assumes you have a solid understanding of how to use Pagefind conventionally. Read through the [standard Pagefind documentation](https://pagefind.app/) first, if you haven't.
+
+***
+
+Using this indexing library is handy if you're integrating Pagefind directly into a static site generator, or for complex tasks like indexing JSON files or other non-HTML sources of content.
+
+Example usage:
+
+```js
+import * as pagefind from "pagefind";
+
+// Create a Pagefind search index to work with
+const { index } = await pagefind.createIndex();
+
+// Add content to it
+await index.addHTMLFile({
+    path: "my_file/index.html",
+    content: "<html><body><h1>Testing, testing</h1></body></html>"
+});
+
+// Get the index in-memory
+await index.getFiles();
+
+// Write the index to disk
+await index.writeFiles({
+    bundlePath: "./public/_pagefind"
+});
+```
+
+All interations with Pagefind are asynchronous, as they communicate with the native Pagefind binary in the background.
+
+### `pagefind.createIndex`
+
+Creates a Pagefind index that files can be added to.
+The index object returned is unique, and multiple calls to `pagefind.createIndex()` can be made without conflicts.
+
+```js
+import * as pagefind from "pagefind";
+
+const { index } = await pagefind.createIndex();
+
+// ... do things with `index`
+```
+
+`createIndex` optionally takes a configuration object that can apply parts of the [Pagefind CLI config](https://pagefind.app/docs/config-options/). The options available at this level are:
+
+```js
+const { index } = await pagefind.createIndex({
+    rootSelector: "html",
+    excludeSelectors: [".my-code-blocks"],
+    forceLanguage: "en",
+    keepIndexUrl: false,
+    verbose: false,
+    logfile: "debug.log"
+});
+```
+
+See the relevant documentation of each configuration option in the [Configuring the Pagefind CLI](https://pagefind.app/docs/config-options/) documentation.
+
+### index.addDirectory
+
+Indexes a directory from disk using the standard Pagefind indexing behaviour. This is the same action as running the Pagefind binary with `--source <dir>`. It can be handy to run this indexing step, and then add custom non-HTML records to the index before writing it to disk.
+
+```js
+const { errors, page_count } = await index.addDirectory({
+    path: "public",
+    glob: "**/*.{html}" // optional
+});
+```
+
+If relative, `path` will be relative to the current working directory of your Node process.
+
+Optionally, a custom `glob` can be supplied, which controls which files Pagefind will consume within the directory. The default is shown, and the `glob` option can be omitted entirely.
+
+A response with an `errors` array containing error messages indicates that Pagefind failed to process this directory.
+If successful, `page_count` will be the number of pages that were added to the index.
+
+### index.addHTMLFile
+
+Adds a virtual HTML file to the Pagefind index. Useful for files that don't exist on disk, for example a static site generator that is serving files from memory.
+
+```js
+const { errors, file } = await index.addHTMLFile({
+    path: "contact/index.html",
+    content: "<html><body> <h1>A Full HTML Document</h1> <p> . . . </p> </body></html>"
+});
+```
+
+The `path` here should represent the output path of this HTML file if it were to exist on disk. Pagefind will use this path to generate the URL.
+
+The `content` should be the full HTML source, including the outer `<html> </html>` tags. This will be run through Pagefind's standard HTML indexing process, and should contain any required Pagefind attributes to control behaviour.
+
+A response with an `errors` array containing error messages indicates that Pagefind failed to index this content.
+If successful, the `file` object is returned containing some metadata about the completed indexing.
+
+### index.addCustomRecord
+
+Adds a direct virtual record to the Pagefind index. Useful for adding non-HTML content to the index.
+
+```js
+const { errors, file } = await index.addHTMLFile({
+    url: "/contact/",
+    content: "My raw content to be indexed for search. Will be lightly processed by Pagefind.",
+    language: "en",
+    meta: {
+        title: "Contact",
+        category: "Landing Page"
+    },
+    filters: {
+        tags: ["landing", "company"]
+    },
+    sort: {
+        weight: "20"
+    }
+});
+```
+
+The `url`, `content`, and `language` fields are all required. `language` should be an [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes).
+
+`meta` is strictly a flat object of keys to string values. See the [Metadata documentation](https://pagefind.app/docs/metadata/) for semantics.
+
+`filters` is strictly a flat object of keys to arrays of string values. See the [Filters documentation](https://pagefind.app/docs/filtering/) for semantics.
+
+`sort` is strictly a flat object of keys to string values. See the [Sort documentation](https://pagefind.app/docs/sorts/) for semantics. *When Pagefind is processing an index, number-like strings will be sorted numerically rather than alphabetically.*
+
+A response with an `errors` array containing error messages indicates that Pagefind failed to index this content.
+If successful, the `file` object is returned containing some metadata about the completed indexing.
+
+### index.getFiles
+
+Get buffers of all files in the Pagefind index. Useful for integrating a Pagefind index into the development mode of a static site generator and hosting these files yourself.
+
+```js
+const { errors, files } = await index.getFiles();
+
+for (const file of files) {
+    console.log(file.path);
+    // do something with file.content
+}
+```
+
+A response with an `errors` array containing error messages indicates that Pagefind failed to action this request.
+
+If successful, `files` will be an array containing file objects. Each object contains a `path` key, which is the URL this file should be served at, and a `content` key containing the raw Buffer of this file.
+
+### index.writeFiles
+
+Writes the index files to disk, as they would be written when running the standard Pagefind binary directly.
+
+```js
+const { errors } = await index.writeFiles({
+    bundlePath: "./public/_pagefind"
+});
+```
+
+The `bundlePath` option should contain the path to the desired Pagefind bundle directory. If relative, is relative to the current working directory of your Node process.
+
+A response with an `errors` array containing error messages indicates that Pagefind failed to action this request.
+
+### index.deleteIndex
+
+Deletes the data for the given index from the Pagefind binary service. Doesn't affect any written files or Buffers returned by `getFiles()`.
+
+```js
+await index.deleteIndex();
+```
+
+Calling `index.getFiles()` or `index.writeFiles()` doesn't consume the index, and further modifications can be made. In situations where many indexes are being created, the `deleteIndex` call helps your clear out memory from the Pagefind binary service.
+
+Reusing the `index` object you called this on will cause errors to be returned.
+
+Not calling this method is fine — these indexes will be cleaned up when your Node process exits.

types/index.d.ts

 /**
  * Create a new Pagefind index that files can be added to
  */
-export function createIndex(): Promise<NewIndexResponse>;
+export function createIndex(config: PagefindServiceConfig): Promise<NewIndexResponse>;
 
+export interface PagefindServiceConfig {
+    /** 
+     * The element Pagefind should treat as the root of the document, defaults to `html`.
+     * Usually you will want to use the data-pagefind-body attribute instead.
+     * @example ".my-html-outer"
+     */
+    rootSelector?: string,
+    /** 
+     * Custom selectors that Pagefind should ignore when indexing.
+     * Usually you will want to use the data-pagefind-ignore attribute instead.
+     * @example ["svg", ".my-code-blocks"]
+     */
+    excludeSelectors?: string[],
+    /**
+     * Ignore any detected languages and index the whole site as a single language.
+     * Expects an ISO 639-1 code.
+     */
+    forceLanguage?: string,
+    /**
+     * Print verbose logging while indexing the site. Does not impact the web-facing search.
+     * When running as a service, only impacts the logfile (if present).
+     */
+    verbose?: boolean,
+    /**
+     * Path to a logfile to write to. Will replace the file on each run.
+     */
+    logfile?: string,
+    /**
+     * Keep `index.html` at the end of search result paths.
+     * Defaults to false, stripping `index.html`.
+     */
+    keepIndexUrl?: boolean,
+}
+
+
 export interface NewIndexResponse {
@@ -17,4 +52,6 @@ errors: string[],
     addCustomRecord: typeof addCustomRecord,
+    addDirectory: typeof addDirectory,
     writeFiles: typeof writeFiles,
     getFiles: typeof getFiles,
+    deleteIndex: typeof deleteIndex,
 }
@@ -30,2 +67,6 @@
 declare function addCustomRecord(record: CustomRecord): Promise<NewFileResponse>;
+/**
+ * Index a directory of HTML files from disk
+ */
+declare function addDirectory(path: SiteDirectory): Promise<IndexingResponse>;
 
@@ -65,3 +106,3 @@ /**
     content: string,
-    /** What language is this record written in. Multiple languages will be split into separate indexes */
+    /** What language is this record written in. Multiple languages will be split into separate indexes. Expects an ISO 639-1 code. */
     language: string,
@@ -76,2 +117,25 @@ /** The metadata to attach to this record. Supplying a `title` is highly recommended */
 
+/**
+ * The data required for Pagefind to index the files in a directory
+ * @example
+ * {
+ *   path: "public",
+ *   glob: "**\/*.{html}"
+ * }
+ */
+export interface SiteDirectory {
+    /**
+     * The path to the directory to index.
+     * If relative, is relative to the cwd.
+     */
+    path: string,
+    /** Optionally, a custom glob to evaluate for finding files. Default to all HTML files. */
+    glob?: string
+}
+
+export interface IndexingResponse {
+    errors: string[],
+    page_count: number
+}
+
 export interface NewFileResponse {
@@ -92,9 +156,22 @@ errors: string[],
 /**
- * Write the index files to the cwd
+ * Write the index files to disk
  */
-declare function writeFiles(): Promise<WriteFilesResponse>;
+declare function writeFiles(options?: WriteOptions): Promise<WriteFilesResponse>;
 
+/**
+ * Options for writing a Pagefind index to disk
+ */
+export interface WriteOptions {
+    /** 
+     * The path of the pagefind bundle directory to write to disk.
+     * If relative, is relative to the cwd.
+     * @example "./public/_pagefind"
+     */
+    bundlePath: string
+}
+
+
 export interface WriteFilesResponse {
     errors: string[],
-    bundleLocation: string
+    bundlePath: string
 }
@@ -115,2 +192,7 @@
     content: Buffer
-}
+}
+
+/**
+ * Delete this index and clear it from memory
+ */
+declare function deleteIndex(): Promise<null>;

types/internal.d.ts

@@ -7,3 +7,3 @@ // Requests to the backend.
 export interface InternalServiceRequest {
-    message_id: number,
+    message_id?: number,
     payload: InternalRequestPayload
@@ -15,8 +15,25 @@ }
  */
-export type InternalRequestPayload = InternalNewIndexRequest | InternalAddFileRequest | InternalAddRecordRequest | InternalWriteFilesRequest | InternalGetFilesRequest;
+export type InternalRequestPayload =
+  | InternalNewIndexRequest
+  | InternalAddFileRequest
+  | InternalAddRecordRequest
+  | InternalAddDirRequest
+  | InternalWriteFilesRequest
+  | InternalGetFilesRequest
+  | InternalDeleteIndexRequest;
 
 export interface InternalNewIndexRequest {
-    type: 'NewIndex'
+    type: 'NewIndex',
+    config?: InternalPagefindServiceConfig
 }
 
+export interface InternalPagefindServiceConfig {
+    root_selector?: string,
+    exclude_selectors?: string[],
+    force_language?: string,
+    verbose?: boolean,
+    logfile?: string,
+    keep_index_url?: boolean,
+}
+
 export interface InternalAddFileRequest {
@@ -30,3 +47,3 @@ type: 'AddFile',
 export interface InternalAddRecordRequest {
-    type: 'AddRecord'
+    type: 'AddRecord',
     index_id: number,
@@ -41,5 +58,13 @@ url: string,
 
+export interface InternalAddDirRequest {
+    type: 'AddDir',
+    index_id: number,
+    path: string,
+    glob?: string
+}
+
 export interface InternalWriteFilesRequest {
     type: 'WriteFiles',
-    index_id: number
+    index_id: number,
+    bundle_path?: string
 }
@@ -52,2 +77,7 @@
 
+export interface InternalDeleteIndexRequest {
+    type: 'DeleteIndex',
+    index_id: number
+}
+
 // Responses from the backend.
@@ -68,2 +98,3 @@
     type: 'Error',
+    original_message?: string,
     message: string
@@ -75,3 +106,9 @@ }
  */
-export type InternalResponsePayload = InternalNewIndexResponse | InternalIndexedFileResponse | InternalWriteFilesResponse | InternalGetFilesResponse;
+export type InternalResponsePayload = 
+  | InternalNewIndexResponse
+  | InternalIndexedFileResponse
+  | InternalIndexedDirResponse
+  | InternalWriteFilesResponse
+  | InternalGetFilesResponse
+  | InternalDeleteIndexResponse;
 
@@ -90,5 +127,10 @@ export interface InternalNewIndexResponse {
 
+export interface InternalIndexedDirResponse {
+    type: 'IndexedDir',
+    page_count: number
+}
+
 export interface InternalWriteFilesResponse {
     type: 'WriteFiles',
-    bundle_location: string,
+    bundle_path: string,
 }
@@ -106,2 +148,6 @@
 
+export interface InternalDeleteIndexResponse {
+    type: 'DeleteIndex'
+}
+
 /**
@@ -108,0 +154,0 @@ * What the service returns to the wrapping javascript detailing a response

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

40799

increased by60.46%

Lines of code

831

increased by33.17%

Number of lines in readme file

197

increased by4825%

No dependency changes