astro-robots - npm Package Compare versions

Comparing version 1.0.2 to 2.0.0

dist/consts.d.ts

 export type SearchEngines = {
-  360: '360Spider' | '360Spider-Image' | '360Spider-Video' | 'HaoSouSpider';
-  Apple: 'Applebot' | 'AppleNewsBot';
-  Baidu: 'Baiduspider' | 'Baiduspider-image' | 'Baiduspider-mobile' | 'Baiduspider-news' | 'Baiduspider-video';
-  Bing: 'bingbot' | 'BingPreview' | 'msnbot' | 'msnbot-media' | 'adidxbot' | 'MSN';
-  Bublup: 'BublupBot';
-  Cliqz: 'Cliqzbot';
-  Coccoc: 'coccoc' | 'coccocbot-image' | 'coccocbot-web';
-  Daumoa: 'Daumoa';
-  Dazoo: 'DeuSu';
-  Duckduckgo: 'DuckDuckBot' | 'DuckDuckGo-Favicons-Bot';
-  Eurip: 'EuripBot';
-  Exploratodo: 'Exploratodo';
-  Findx: 'Findxbot';
-  Goo: 'gooblog' | 'ichiro';
-  Google: 'Googlebot' | 'Googlebot-Image' | 'Googlebot-Mobile' | 'Googlebot-News' | 'Googlebot-Video' | 'Mediapartners-Google' | 'AdsBot-Google' | 'AdsBot-Google-Mobile' | 'AdsBot-Google-Mobile-Apps' | 'Mediapartners-Google' | 'Storebot-Google' | 'Google-InspectionTool' | 'FeedFetcher-Google';
-  Istella: 'istellabot';
-  Jike: 'JikeSpider';
-  Lycos: 'Lycos';
-  Mail: 'Mail.Ru';
-  Mojeek: 'MojeekBot';
-  Orange: 'OrangeBot';
-  Botje: 'Plukkie';
-  Qwant: 'Qwantify';
-  Rambler: 'Rambler';
-  Seznam: 'SeznamBot';
-  Soso: 'Sosospider';
-  Yahoo: 'Slurp';
-  Sogou: 'Sogou blog' | 'Sogou inst spider' | 'Sogou News Spider' | 'Sogou Orion spider' | 'Sogou spider2' | 'Sogou web spider';
-  Sputnik: 'SputnikBot';
-  Ask: 'Teoma';
-  Wortbox: 'wotbox';
-  Yandex: 'Yandex' | 'YandexMobileBot';
-  Naver: 'Yeti';
-  Yioop: 'YioopBot';
-  Yooz: 'yoozBot';
-  Youdao: 'YoudaoBot';
+    360: '360Spider' | '360Spider-Image' | '360Spider-Video' | 'HaoSouSpider';
+    Apple: 'Applebot' | 'AppleNewsBot';
+    Baidu: 'Baiduspider' | 'Baiduspider-image' | 'Baiduspider-mobile' | 'Baiduspider-news' | 'Baiduspider-video';
+    Bing: 'bingbot' | 'BingPreview' | 'msnbot' | 'msnbot-media' | 'adidxbot' | 'MSN';
+    Bublup: 'BublupBot';
+    Cliqz: 'Cliqzbot';
+    Coccoc: 'coccoc' | 'coccocbot-image' | 'coccocbot-web';
+    Daumoa: 'Daumoa';
+    Dazoo: 'DeuSu';
+    Duckduckgo: 'DuckDuckBot' | 'DuckDuckGo-Favicons-Bot';
+    Eurip: 'EuripBot';
+    Exploratodo: 'Exploratodo';
+    Findx: 'Findxbot';
+    Goo: 'gooblog' | 'ichiro';
+    Google: 'Googlebot' | 'Googlebot-Image' | 'Googlebot-Mobile' | 'Googlebot-News' | 'Googlebot-Video' | 'Mediapartners-Google' | 'AdsBot-Google' | 'AdsBot-Google-Mobile' | 'AdsBot-Google-Mobile-Apps' | 'Mediapartners-Google' | 'Storebot-Google' | 'Google-InspectionTool' | 'FeedFetcher-Google';
+    Istella: 'istellabot';
+    Jike: 'JikeSpider';
+    Lycos: 'Lycos';
+    Mail: 'Mail.Ru';
+    Mojeek: 'MojeekBot';
+    Orange: 'OrangeBot';
+    Botje: 'Plukkie';
+    Qwant: 'Qwantify';
+    Rambler: 'Rambler';
+    Seznam: 'SeznamBot';
+    Soso: 'Sosospider';
+    Yahoo: 'Slurp';
+    Sogou: 'Sogou blog' | 'Sogou inst spider' | 'Sogou News Spider' | 'Sogou Orion spider' | 'Sogou spider2' | 'Sogou web spider';
+    Sputnik: 'SputnikBot';
+    Ask: 'Teoma';
+    Wortbox: 'wotbox';
+    Yandex: 'Yandex' | 'YandexMobileBot';
+    Naver: 'Yeti';
+    Yioop: 'YioopBot';
+    Yooz: 'yoozBot';
+    Youdao: 'YoudaoBot';
 };
 export type SocialNetwork = {
-  Facebook: 'facebookcatalog' | 'facebookexternalhit' | 'Facebot';
-  Pinterest: 'Pinterest';
-  Tittwer: 'Twitterbot';
-  WhatsApp: 'WhatsApp';
-  LinkedIn: 'LinkedInBot';
+    Facebook: 'facebookcatalog' | 'facebookexternalhit' | 'Facebot';
+    Pinterest: 'Pinterest';
+    Tittwer: 'Twitterbot';
+    WhatsApp: 'WhatsApp';
+    LinkedIn: 'LinkedInBot';
 };
 export type SearchEngineOptimization = {
-  Ahrefs: 'AhrefsBot';
-  Moz: 'Moz dotbot' | 'Moz rogerbot';
-  WebMeUp: 'BLEXBot';
-  Botify: 'Botify';
-  Babbar: 'Barkrowler';
-  SEMrush: 'SEMrush' | 'SemrushBotSI';
-  Cxense: 'Cxense';
-  EzoicInc: 'EzoicBot';
-  DataForSEO: 'DataForSEO';
-  PrerenderLLC: 'prerender';
+    Ahrefs: 'AhrefsBot';
+    Moz: 'Moz dotbot' | 'Moz rogerbot';
+    WebMeUp: 'BLEXBot';
+    Botify: 'Botify';
+    Babbar: 'Barkrowler';
+    SEMrush: 'SEMrush' | 'SemrushBotSI';
+    Cxense: 'Cxense';
+    EzoicInc: 'EzoicBot';
+    DataForSEO: 'DataForSEO';
+    PrerenderLLC: 'prerender';
 };
 export type UsertAgentType = '*' | SearchEngines[keyof SearchEngines] | SocialNetwork[keyof SocialNetwork] | SearchEngineOptimization[keyof SearchEngineOptimization];

dist/index.js

@@ -1,27 +0,232 @@
-import x from"node:fs";var g="astro-robots";var c=class{colors={reset:"\x1B[0m",fg:{red:"\x1B[31m",green:"\x1B[32m",yellow:"\x1B[33m",cyanBold:"\x1B[1m\x1B[36m"}};packageName;constructor(r){this.packageName=r}log(r,o=""){let a=r.join(`
-`),s=new Date,e=s.getHours().toString().padStart(2,"0"),n=s.getMinutes().toString().padStart(2,"0"),m=s.getSeconds().toString().padStart(2,"0"),f=`\x1B[2m${e}:${n}:${m}\x1B[22m`;console.log(`${f} %s[${this.packageName}]%s ${a}`,o,o?this.colors.reset:"")}info(...r){this.log(r,this.colors.fg.cyanBold)}success(...r){this.log(r,this.colors.fg.cyanBold)}warn(...r){this.log([`${this.colors.fg.yellow}(!)${this.colors.reset} ${r}`],this.colors.fg.cyanBold)}error(...r){this.log([`${this.colors.fg.red}failed!${this.colors.reset}`,...r],this.colors.fg.cyanBold)}};function u(t){let r=performance.now();t();let o=performance.now();return Math.floor(o-r)}var l=new c(g);function p(t){let r=/^(?=.{1,253}$)(?:(?!-)[a-zA-Z0-9-]{1,63}(?<!-)\.)+[a-zA-Z]{2,63}$/;typeof t!="string"&&i("Host must be a string","error"),r.test(t)||i("Host is invalid","error")}function h(t){let r="";return t.host===!0||t.host===!1||(typeof t.host=="number"?p(t.host):typeof t.host=="string"&&t.host!=="localhost"&&(p(t.host),r+=`Host: ${t.host}
-`)),r}function b(t){/^https?:\/\/[^\s/$.?#].[^\s]*\.(xml|txt|html|xml.gz|txt.gz|json|xhtml)$/i.test(t)||i("sitemap [URL is invalid or not a valid sitemap file.]",!0)}function y(t,r){let o="";if(t.sitemap===!0)o+=`Sitemap: ${r}sitemap-index.xml
-`;else if(typeof t.sitemap=="number")i("sitemap [URL is invalid or not a valid sitemap file.]",!0);else if(typeof t.sitemap=="string")b(t.sitemap),o+=`Sitemap: ${t.sitemap}
-`;else if(Array.isArray(t.sitemap))for(let a of t.sitemap)b(a),o+=`Sitemap: ${a}
-`;return o}function i(t,r){let o="\x1B[1mRefer:\x1B[22m",a=e=>{l.info("\x1B[31mFailured!\x1B[39m")},s=e=>{l.warn(`Skiped! [${e}].`)};switch(r){case"warn":s(t);break;case"error":throw a(t),new Error(`${t}`);case!0:throw a(t),new Error(`${t}
-${o}
-  Visit \x1B[4mhttps://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt#useful-robots.txt-rules\x1B[24m for instructions.`);default:throw a(t),new Error(`${t}
-${o}
-  Visit \x1B[4mhttps://yandex.com/support/webmaster/controlling-robot/robots-txt.html#recommend\x1B[24m for instructions.`)}}function w(t,r){var a;let o="";for(let s of t.policy??[]){if((a=t==null?void 0:t.policy)==null||a.forEach((e,n)=>{var m,f;e.userAgent||i(`policy[${n}].userAgent [Required, one or more per group].
-${JSON.stringify(e,null,2)}`,!!e.userAgent),(!e.allow&&!e.disallow||((m=e.allow)==null?void 0:m.length)===0&&((f=e.disallow)==null?void 0:f.length)===0)&&i(`policy[${n}] [At least one or more 'disallow' or 'allow' entries per rule].
-${JSON.stringify(e,null,2)}`,!e.allow&&!e.disallow),e.crawlDelay&&typeof e.crawlDelay!="number"?i(`policy[${n}].crawlDelay [Must be number].
-${JSON.stringify(e,null,2)}`,!1):e.crawlDelay!==void 0&&(e==null?void 0:e.crawlDelay)<0?i(`policy[${n}].crawlDelay [Must be a positive number].
-${JSON.stringify(e,null,2)}`,!1):e.crawlDelay!==void 0&&((e==null?void 0:e.crawlDelay)<.1||e.crawlDelay>60)&&i(`policy[${n}].crawlDelay [Must be between 0.1 and 60 seconds].
-${JSON.stringify(e,null,2)}`,!1)}),s.userAgent){let e=Array.isArray(s.userAgent)?s.userAgent:[s.userAgent||"*"];for(let n of e)n&&(o+=`User-agent: ${n}
-`)}if(s.allow){let e=Array.isArray(s.allow)?s.allow:[s.allow];for(let n of e)o+=`Allow: ${n}
-`}if(s.disallow){let e=Array.isArray(s.disallow)?s.disallow:[s.disallow];for(let n of e)o+=`Disallow: ${n}
-`}if(s.crawlDelay&&(o+=`Crawl-delay: ${s.crawlDelay}
-`),s.cleanParam){let e=Array.isArray(s.cleanParam)?s.cleanParam:[s.cleanParam];for(let n of e)o+=`Clean-param: ${n}
-`}t.policy&&s!==t.policy[t.policy.length-1]?o+=`
-`:t.sitemap!==!1&&(o+=`
+// src/index.ts
+import fs from "fs";
+
+// src/data/pkg-name.ts
+var packageName = "astro-robots";
+
+// src/core.ts
+function validateHost(host, logger) {
+  const hostPattern = /^(?=.{1,253}$)(?:(?!-)[a-zA-Z0-9-]{1,63}(?<!-)\.)+[a-zA-Z]{2,63}$/;
+  if (typeof host !== "string") {
+    throwMsg("Host must be a string", "error", logger);
+  }
+  if (!hostPattern.test(host)) {
+    throwMsg("Host is invalid", "error", logger);
+  }
+}
+function generateHostContent(config, logger) {
+  let content = "";
+  if (config.host === true) {
+  } else if (config.host === false) {
+  } else if (typeof config.host === "number") {
+    validateHost(config.host, logger);
+  } else if (typeof config.host === "string" && config.host !== "localhost") {
+    validateHost(config.host, logger);
+    content += `Host: ${config.host}
+`;
+  }
+  return content;
+}
+function validateUrl(url, logger) {
+  const urlPattern = /^https?:\/\/[^\s/$.?#].[^\s]*\.(xml|txt|html|xml.gz|txt.gz|json|xhtml)$/i;
+  if (!urlPattern.test(url)) {
+    throwMsg(`sitemap [URL is invalid or not a valid sitemap file.]`, true, logger);
+  }
+}
+function generateSitemapContent(config, siteHref, logger) {
+  let content = "";
+  if (config.sitemap === true) {
+    content += `Sitemap: ${siteHref}sitemap-index.xml
+`;
+  } else if (typeof config.sitemap === "number") {
+    throwMsg(`sitemap [URL is invalid or not a valid sitemap file.]`, true, logger);
+  } else if (typeof config.sitemap === "string") {
+    validateUrl(config.sitemap, logger);
+    content += `Sitemap: ${config.sitemap}
+`;
+  } else if (Array.isArray(config.sitemap)) {
+    for (let url of config.sitemap) {
+      validateUrl(url, logger);
+      content += `Sitemap: ${url}
+`;
+    }
+  }
+  return content;
+}
+function throwMsg(msg, type, logger) {
+  const sentenceHead = `\x1B[1mRefer:\x1B[22m`;
+  const failured = (message) => {
+    logger.info(`\x1B[31mFailured! [${message}]\x1B[39m`);
+  };
+  const warn = (message) => {
+    logger.warn(`Skiped! [${message}].`);
+  };
+  switch (type) {
+    case "warn":
+      warn(msg);
+      break;
+    case "error":
+      failured(msg);
+      throw new Error(`${msg}`);
+    case true:
+      failured(msg);
+      throw new Error(`${msg}
+${sentenceHead}
+  Visit \x1B[4m${`https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt#useful-robots.txt-rules`}\x1B[24m for instructions.`);
+    default:
+      failured(msg);
+      throw new Error(`${msg}
+${sentenceHead}
+  Visit \x1B[4m${`https://yandex.com/support/webmaster/controlling-robot/robots-txt.html#recommend`}\x1B[24m for instructions.`);
+  }
+}
+function generateContent(config, siteMapHref, logger) {
+  var _a;
+  let content = "";
+  for (const policy of config.policy ?? []) {
+    (_a = config == null ? void 0 : config.policy) == null ? void 0 : _a.forEach((policy2, index) => {
+      var _a2, _b;
+      if (!policy2.userAgent) {
+        throwMsg(
+          `policy[${index}].userAgent [Required, one or more per group].
+${JSON.stringify(policy2, null, 2)}`,
+          !!policy2.userAgent,
+          logger
+        );
+      }
+      if (!policy2.allow && !policy2.disallow || ((_a2 = policy2.allow) == null ? void 0 : _a2.length) === 0 && ((_b = policy2.disallow) == null ? void 0 : _b.length) === 0) {
+        throwMsg(
+          `policy[${index}] [At least one or more 'disallow' or 'allow' entries per rule].
+${JSON.stringify(policy2, null, 2)}`,
+          !policy2.allow && !policy2.disallow,
+          logger
+        );
+      }
+      if (policy2.crawlDelay && typeof policy2.crawlDelay !== "number") {
+        throwMsg(`policy[${index}].crawlDelay [Must be number].
+${JSON.stringify(policy2, null, 2)}`, false, logger);
+      } else if (policy2.crawlDelay !== void 0 && (policy2 == null ? void 0 : policy2.crawlDelay) < 0) {
+        throwMsg(`policy[${index}].crawlDelay [Must be a positive number].
+${JSON.stringify(policy2, null, 2)}`, false, logger);
+      } else if (policy2.crawlDelay !== void 0 && ((policy2 == null ? void 0 : policy2.crawlDelay) < 0.1 || policy2.crawlDelay > 60)) {
+        throwMsg(`policy[${index}].crawlDelay [Must be between 0.1 and 60 seconds].
+${JSON.stringify(policy2, null, 2)}`, false, logger);
+      }
+    });
+    if (policy.userAgent) {
+      const userAgents = Array.isArray(policy.userAgent) ? policy.userAgent : [policy.userAgent || "*"];
+      for (const userAgent of userAgents) {
+        if (userAgent) {
+          content += `User-agent: ${userAgent}
+`;
+        }
+      }
+    }
+    if (policy.allow) {
+      const allowPaths = Array.isArray(policy.allow) ? policy.allow : [policy.allow];
+      for (const path of allowPaths) {
+        content += `Allow: ${path}
+`;
+      }
+    }
+    if (policy.disallow) {
+      const disallowPaths = Array.isArray(policy.disallow) ? policy.disallow : [policy.disallow];
+      for (const path of disallowPaths) {
+        content += `Disallow: ${path}
+`;
+      }
+    }
+    if (policy.crawlDelay) {
+      content += `Crawl-delay: ${policy.crawlDelay}
+`;
+    }
+    if (policy.cleanParam) {
+      const cleanParams = Array.isArray(policy.cleanParam) ? policy.cleanParam : [policy.cleanParam];
+      for (const param of cleanParams) {
+        content += `Clean-param: ${param}
+`;
+      }
+    }
+    if (config.policy && policy !== config.policy[config.policy.length - 1]) {
+      content += `
+`;
+    } else if (config.sitemap !== false) {
+      content += `
 # crawling rule(s) for above bots
-`)}return o+=y(t,r),o+=h(t),o}function d(t){t>10&&(console.log(`
-\x1B[42m generating 'robots.txt' file \x1B[0m`),console.log(`${[`
-\x1B[33m(!) Keep your 'robots.txt' file size under 10 KB for best crawling results.`,"- To keep it low, only include directives that are necessary for your site.",`- Remove rules for pages that no longer exist to avoid bloat.\x1B[0m
-`].join(`
-`)}`))}function $(t){return x.statSync(t).size/1024}var A={sitemap:!0,host:!1,policy:[{userAgent:"*",allow:"/"}]};function S(t){let r,o,a,s={...A,...t};return{name:g,hooks:{"astro:config:setup":({config:e})=>{r=e},"astro:build:start":()=>{o=new URL(r.base,r.site).href},"astro:build:done":async({dir:e})=>{a=u(()=>{x.writeFileSync(new URL("robots.txt",e),w(s,o),"utf-8")});let n=$(new URL("robots.txt",e));l.info(`\x1B[2mCompleted in ${a}ms\x1B[22m`),d(n),l.success(`\x1B[32mgenerated\x1B[0m 'robots.txt' ${n} KB`)}}}}export{S as default};
+`;
+    }
+  }
+  content += generateSitemapContent(config, siteMapHref, logger);
+  content += generateHostContent(config, logger);
+  return content;
+}
+function printInfo(fileSize, executionTime, logger) {
+  logger.info(`\x1B[2mCompleted in ${executionTime}ms\x1B[22m`);
+  if (fileSize > 10) {
+    console.log(`
+\x1B[42m\x1B[30m generating 'robots.txt' file \x1B[39m\x1B[0m`);
+    const warnMsg = [
+      `
+\x1B[33m(!) Keep your 'robots.txt' file size under 10 KB for best crawling results.`,
+      `- To keep it low, only include directives that are necessary for your site.`,
+      `- Remove rules for pages that no longer exist to avoid bloat.\x1B[0m
+`
+    ];
+    console.log(`${warnMsg.join("\n")}`);
+  }
+  logger.info(`\x1B[32mgenerated\x1B[0m 'robots.txt' ${fileSize} KB`);
+}
+
+// src/utils/measureExecutionTime.ts
+function measureExecutionTime(callback) {
+  const startTime = performance.now();
+  callback();
+  const endTime = performance.now();
+  const executionTime = Math.floor(endTime - startTime);
+  return executionTime;
+}
+
+// src/index.ts
+function getFileSizeInKilobytes(filename) {
+  const stats = fs.statSync(filename);
+  const fileSizeInBytes = stats.size;
+  const fileSizeInKilobytes = fileSizeInBytes / 1024;
+  return fileSizeInKilobytes;
+}
+var defaultConfig = {
+  sitemap: true,
+  host: false,
+  policy: [
+    {
+      userAgent: "*",
+      allow: "/"
+    }
+  ]
+};
+function createRobotsIntegration(astroConfig) {
+  let config;
+  let finalSiteMapHref;
+  let executionTime;
+  const megeredConfig = { ...defaultConfig, ...astroConfig };
+  return {
+    name: packageName,
+    hooks: {
+      "astro:config:setup": ({ config: cfg }) => {
+        config = cfg;
+      },
+      "astro:build:start": () => {
+        finalSiteMapHref = new URL(config.base, config.site).href;
+      },
+      "astro:build:done": async ({ dir, logger }) => {
+        executionTime = measureExecutionTime(() => {
+          fs.writeFileSync(new URL("robots.txt", dir), generateContent(megeredConfig, finalSiteMapHref, logger), "utf-8");
+        });
+        const fileSize = getFileSizeInKilobytes(new URL("robots.txt", dir));
+        printInfo(fileSize, executionTime, logger);
+      }
+    }
+  };
+}
+export {
+  createRobotsIntegration as default
+};

package.json

@@ -5,3 +5,3 @@ {
   "description": "The lightweight robots.txt generator makes it simple to create compliant robots.txt files specifically for Astro integrations, with zero-config support, an intuitive JSDoc API, and always up-to-date verified bots user-agents.",
-  "version": "1.0.2",
+  "version": "2.0.0",
   "type": "module",
@@ -20,3 +20,3 @@ "exports": {
     "clear": "rm -rf ./dist",
-    "build": "npm run clear && at-scripts create-pkg-name && at-scripts build && tsc -p tsconfig.build.json"
+    "build": "at-scripts create-pkg-name && at-scripts build && tsc -p tsconfig.build.json"
   },
@@ -45,5 +45,6 @@ "keywords": [
   "devDependencies": {
-    "astro": "^3.0.6",
-    "typescript": "^5.2.2"
+    "@types/node": "^20.10.4",
+    "astro": "^4.0.3",
+    "typescript": "^5.3.3"
   }
 }

README.md

@@ -21,3 +21,3 @@ <div align="center">
 
-> This package is compatible with Astro 2.0 and above, which support the Integrations API.
+> This package is compatible with **Astro 3.0.0** and above, due to the use of AstroIntegrationLogger.
 
@@ -24,0 +24,0 @@ To automate the installation, use the `astro add` command-line tool. You can initialize it via `npx`, `yarn`, or `pnpm` based on your preference.

New alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

21743

increased by16.03%

Lines of code

417

increased by87.84%

Worsened metrics

Dev dependency count

3

increased by50%

Number of low supply chain risk alerts

1

increased byInfinity%

No dependency changes