astro-robots-txt - npm Package Compare versions

Comparing version 0.2.0 to 0.3.0

dist/index.d.ts

@@ -14,3 +14,3 @@ import type { AstroIntegration } from 'astro';
 } | undefined;
-declare const createPlugin: (pluginOptions: RobotsTxtOptions) => AstroIntegration;
+declare const createPlugin: (options?: RobotsTxtOptions) => AstroIntegration;
 export default createPlugin;

dist/index.js

@@ -0,1 +1,21 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+
+// src/index.ts
+import { ZodError } from "zod";
+
 // ../utils/src/is-object-empty.ts
@@ -90,2 +110,39 @@ var isObjectEmpty = (o) => {
 
+// ../utils/src/is-file-exists.ts
+import fs from "fs";
+async function isFileExists(file) {
+  try {
+    const result = await fs.promises.stat(file);
+    return result.isFile();
+  } catch (e) {
+    if (e.code !== "ENOENT") {
+      throw e;
+    }
+    return false;
+  }
+}
+
+// ../utils/src/load-config.ts
+async function loadConfig(name, base) {
+  if (!name) {
+    return null;
+  }
+  const exts = ["js", "mjs", "cjs"];
+  for (const ext of exts) {
+    const fileName = `${name}.config.${ext}`;
+    const file = new URL(fileName, base);
+    if (await isFileExists(file)) {
+      const module = await import(file.pathname);
+      if (!module.default) {
+        throw new Error(`'${fileName}' doesn't have default export`);
+      }
+      return module.default;
+    }
+  }
+  return null;
+}
+
+// src/index.ts
+import merge from "deepmerge";
+
 // src/data/package-name.ts
@@ -95,3 +152,3 @@ var packageName = "astro-robots-txt";
 // src/on-build-done.ts
-import fs from "fs";
+import fs2 from "fs";
 
@@ -120,106 +177,36 @@ // src/with-options.ts
 
-// src/is-opts-valid.ts
-var logger;
-var validateSitemapItem = (sitemap) => {
-  if (!isValidUrl(sitemap)) {
-    logger.warn("Option `sitemap` contains not valid url.");
-    return false;
-  }
-  if (!isValidHttpUrl(sitemap)) {
-    logger.warn("Option `sitemap` needs `http` or `https` protocol in url.");
-    return false;
-  }
-  return true;
+// src/validate-opts.ts
+import { z as z2 } from "zod";
+
+// src/schema.ts
+import { z } from "zod";
+var validateSitemapItem = () => z.string().refine((val) => !val || isValidUrl(val), {
+  message: "Not valid url"
+}).refine((val) => !val || isValidHttpUrl(val), {
+  message: "Only `http` or `https` protocol allowed"
+});
+var validateCleanParam = () => z.string().max(500);
+var RobotsTxtOptionsSchema = z.object({
+  host: z.string().optional().refine((val) => !val || isValidHostname(val), {
+    message: "Not valid host"
+  }),
+  sitemap: validateSitemapItem().or(validateSitemapItem().array()).or(z.boolean()).optional().default(true),
+  policy: z.object({
+    userAgent: z.string().min(1),
+    allow: z.string().or(z.string().array()).optional(),
+    disallow: z.string().or(z.string().array()).optional(),
+    cleanParam: validateCleanParam().or(validateCleanParam().array()).optional(),
+    crawlDelay: z.number().nonnegative().refine((val) => typeof val === "undefined" || Number.isFinite(val), { message: "Must be finite number" }).optional()
+  }).array().nonempty().optional()
+});
+
+// src/validate-opts.ts
+var validateOpts = (site, opts) => {
+  const schema = RobotsTxtOptionsSchema.extend({
+    site: z2.string().min(1, {
+      message: "`site` property is required in `astro.config.mjs`."
+    })
+  });
+  schema.parse(__spreadValues({ site: site || "" }, opts || {}));
 };
-var isValidSitemap = (sitemap) => {
-  if (typeof sitemap !== "string" && typeof sitemap !== "boolean" && !Array.isArray(sitemap)) {
-    logger.warn("The robots.txt integration requires `sitemap` option to be string, array of strings or boolean.");
-    return false;
-  }
-  if (sitemap) {
-    if (typeof sitemap === "string") {
-      if (!validateSitemapItem(sitemap)) {
-        return false;
-      }
-    } else if (Array.isArray(sitemap)) {
-      for (const item of sitemap) {
-        if (!validateSitemapItem(item)) {
-          return false;
-        }
-      }
-    }
-  }
-  return true;
-};
-var isValidCleanParamItem = (item) => {
-  if (typeof item !== "string") {
-    logger.warn("String in `cleanParam` option should be a string.");
-    return false;
-  }
-  if (item.length > 500) {
-    logger.warn("String in `cleanParam` option should have no more than 500 characters.");
-    return false;
-  }
-  return true;
-};
-var isValidPolicy = (policy) => {
-  if (!policy) {
-    logger.warn("Options `policy` should be defined.");
-    return false;
-  }
-  if (!Array.isArray(policy)) {
-    logger.warn("Options `policy` must be array.");
-    return false;
-  }
-  if (policy.length === 0) {
-    logger.warn("Options `policy` must be not empty array.");
-    return false;
-  }
-  for (const item of policy) {
-    if (!item.userAgent || item.userAgent.length === 0) {
-      logger.warn("Each `policy` should have a single string `userAgent` option.");
-      return false;
-    }
-    if (item.crawlDelay && typeof item.crawlDelay !== "number" && !Number.isFinite(item.crawlDelay)) {
-      logger.warn("Option `crawlDelay` must be an integer or a float.");
-      return false;
-    }
-    if (item.cleanParam) {
-      if (Array.isArray(item.cleanParam)) {
-        for (const subItem of item.cleanParam) {
-          if (!isValidCleanParamItem(subItem)) {
-            return false;
-          }
-        }
-      } else if (typeof item.cleanParam === "string") {
-        if (!isValidCleanParamItem(item.cleanParam)) {
-          return false;
-        }
-      } else {
-        logger.warn("Option `cleanParam` should be a string or an array.");
-        return false;
-      }
-    }
-  }
-  return true;
-};
-var isOptsValid = (site, opts, _logger) => {
-  logger = _logger;
-  const { host, sitemap, policy } = opts || {};
-  if (!site) {
-    logger.warn("`site` property is required in `astro.config.mjs`.");
-    return false;
-  }
-  if (host && !isValidHostname(host)) {
-    logger.warn("Option `host` does not contain correct host.");
-    return false;
-  }
-  if (sitemap && !isValidSitemap(sitemap)) {
-    return false;
-  }
-  if (policy && !isValidPolicy(policy)) {
-    return false;
-  }
-  return true;
-};
 
@@ -266,7 +253,11 @@ // src/get-robots-txt-content.ts
     }
+    let a = void 0;
     if (Array.isArray(sitemap)) {
-      return sitemap.length > 0 ? sitemap : void 0;
+      a = sitemap;
+    } else if (typeof sitemap === "string") {
+      a = [sitemap];
     }
-    if (typeof sitemap === "string") {
-      return [sitemap];
+    if (a) {
+      a = a.filter(Boolean);
+      return a.length > 0 ? a : void 0;
     }
@@ -292,11 +283,8 @@ }
 // src/on-build-done.ts
-var onBuildDone = (pluginOptions, config, dir, logger2) => {
+var onBuildDone = (pluginOptions, config, dir) => {
   const opts = withOptions(pluginOptions);
-  if (!isOptsValid(config.site, opts, logger2)) {
-    return;
-  }
+  validateOpts(config.site, opts);
   const robotsTxtContent = getRobotsTxtContent(config.site, opts);
   const url = new URL("robots.txt", dir);
-  fs.writeFileSync(url, robotsTxtContent);
-  logger2.success("`robots.txt` is created.");
+  fs2.writeFileSync(url, robotsTxtContent);
 };
@@ -306,3 +294,7 @@ var on_build_done_default = onBuildDone;
 // src/index.ts
-var createPlugin = (pluginOptions) => {
+function formatConfigErrorMessage(err) {
+  const errorList = err.issues.map((issue) => ` ${issue.path.join(".")}  ${issue.message + "."}`);
+  return errorList.join("\n");
+}
+var createPlugin = (options) => {
   let config;
@@ -315,3 +307,18 @@ return {
       },
-      "astro:build:done": async ({ dir }) => on_build_done_default(pluginOptions, config, dir, new Logger(packageName))
+      "astro:build:done": async ({ dir }) => {
+        const namespace = packageName.replace("astro-", "");
+        const external = await loadConfig(namespace, config.root);
+        const merged = merge(external || {}, options || {});
+        const logger = new Logger(packageName);
+        try {
+          on_build_done_default(merged, config, dir);
+          logger.success("`robots.txt` is created.");
+        } catch (err) {
+          if (err instanceof ZodError) {
+            logger.warn(formatConfigErrorMessage(err));
+          } else {
+            throw err;
+          }
+        }
+      }
     }
@@ -318,0 +325,0 @@ };

package.json

 {
   "name": "astro-robots-txt",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Generate a robots.txt for Astro",
@@ -9,5 +9,5 @@ "keywords": [
     "seo",
-    "plugin",
-    "typescript",
-    "robots.txt"
+    "robots.txt",
+    "robotstxt",
+    "robots-txt"
   ],
@@ -35,8 +35,12 @@ "author": {
   ],
+  "dependencies": {
+    "deepmerge": "^4.2.2",
+    "zod": "^3.17.3"
+  },
   "devDependencies": {
-    "@types/node": "^17.0.38",
-    "astro": "^1.0.0-beta.38",
+    "@types/node": "^17.0.40",
+    "astro": "^1.0.0-beta.40",
     "at-scripts": "0.0.3",
     "c8": "^7.11.3",
-    "typescript": "^4.7.2",
+    "typescript": "^4.7.3",
     "vite": "^2.9.9",
@@ -43,0 +47,0 @@ "vitest": "^0.13.1"

README.md

@@ -1,4 +0,2 @@
-<a href="https://bank.gov.ua/en/about/support-the-armed-forces">
-  <img src="https://raw.githubusercontent.com/alextim/help-ukraine-win-flag/master/stop-russian-agressian-help-ukraine-now-link.svg" alt="Help Ukraine now!">
-</a>
+[![Help Ukraine now!](https://raw.githubusercontent.com/alextim/help-ukraine-win-flag/master/stop-russian-agressian-help-ukraine-now-link.svg 'Help Ukraine now!')](https://bank.gov.ua/en/about/support-the-armed-forces)
 
@@ -24,3 +22,3 @@ # astro-robots-txt
 
-**astro-robots-txt** could help in both two cases on the _robots.txt_ side. See details in the demo [repo](https://github.com/alextim/astro-lib/tree/main/examples/robots-txt).
+**astro-robots-txt** could help in both two cases on the _robots.txt_ side. See details in the demo [repo](https://github.com/alextim/astro-lib/tree/main/examples/robots-txt/extended).
 
@@ -122,9 +120,9 @@ ---
 
-|     Name     |         Type          | Required |                   Description                   |
-| :----------: | :-------------------: | :------: | :---------------------------------------------: |
-| `userAgent`  |       `String`        |   Yes    | You must provide name of user agent or wildcard |
-|  `disallow`  | `String` / `String[]` |    No    |                Disallowed paths                 |
-|   `allow`    | `String` / `String[]` |    No    |                  Allowed paths                  |
-| `crawlDelay` |       `Number`        |    No    |                                                 |
-| `cleanParam` | `String` / `String[]` |    No    |                                                 |
+|     Name     |         Type          | Required |                                              Description                                              |
+| :----------: | :-------------------: | :------: | :---------------------------------------------------------------------------------------------------: |
+| `userAgent`  |       `String`        |   Yes    |                            You must provide name of user agent or wildcard                            |
+|  `disallow`  | `String` / `String[]` |    No    |                                       Disallowed paths to index                                       |
+|   `allow`    | `String` / `String[]` |    No    |                                        Allowed paths to index                                         |
+| `crawlDelay` |       `Number`        |    No    | Minimum interval (in secs) for the search robot to wait after loading one page, before starting other |
+| `cleanParam` | `String` / `String[]` |    No    |         Indicates that the page URL contains parameters that should be ignored when indexing          |
 
@@ -195,2 +193,37 @@ **Sample of _astro.config.mjs_**
 
+## Using Configuration Files
+
+You could configure the integration with external file `robots-txt.config.*` (`js`, `cjs`, `mjs`). Put it to the application `root` folder (see about `root` in offical [docs](https://docs.astro.build/en/reference/configuration-reference/)).
+
+The external config must have default export statement:
+
+```js
+// ESM
+export default {
+  ...
+};
+```
+
+or
+
+```js
+// CommonJS
+exports = {
+  ...
+};
+```
+
+:exclamation: Current version of integration doesn't support typescript configs.
+
+### How does the integration internally resolve a config?
+
+| Options parameter provided? | External config exists? | Result                                           |
+| :-------------------------- | :---------------------: | :----------------------------------------------- |
+| No                          |           No            | Default config used                              |
+| Yes                         |           No            | Options parameter used                           |
+| No                          |           Yes           | External config used                             |
+| Yes                         |           Yes           | External config is merged with options parameter |
+
+The external configuration usage example is in the demo [repo](https://github.com/alextim/astro-lib/tree/main/examples/robots-txt/extended).
+
 :exclamation: Important Notes
@@ -197,0 +230,0 @@

No alert changes

Improved metrics

Total package byte prevSize

22052

increased by12.04%

Number of lines in readme file

257

increased by14.73%

Worsened metrics

Dependency count

2

increased byInfinity%

Number of low supply chain risk alerts

2

increased by100%

Dependency changes

• + Addeddeepmerge@^4.2.2

• + Addedzod@^3.17.3

• + Addeddeepmerge@4.3.1(transitive)

• + Addedzod@3.24.2(transitive)