New Case Study:See how Anthropic automated 95% of dependency reviews with Socket.Learn More →
astro-robots-txt
Advanced tools
astro-robots-txt - npm Package Compare versions
Comparing version 0.1.14 to 0.1.15
@@ -7,7 +7,75 @@ // src/index.ts | ||
| // src/utils/isValidHostname.ts | ||
| var isValidHostname = (value) => { | ||
| if (typeof value !== "string") { | ||
| // src/utils/is-object-empty.ts | ||
| var isObjectEmpty = (o) => { | ||
| if (!o) { | ||
| return true; | ||
| } | ||
| if (Array.isArray(o)) { | ||
| return o.length === 0; | ||
| } | ||
| return Object.keys(o).length === 0 && Object.getPrototypeOf(o) === Object.prototype; | ||
| }; | ||
| // src/with-options.ts | ||
| var defaultOptions = { | ||
| host: "", | ||
| sitemap: true, | ||
| policy: [ | ||
| { | ||
| allow: "/", | ||
| userAgent: "*" | ||
| } | ||
| ] | ||
| }; | ||
| var withOptions = (pluginOptions) => { | ||
| if (isObjectEmpty(pluginOptions)) { | ||
| return defaultOptions; | ||
| } | ||
| const options = { | ||
| host: (pluginOptions == null ? void 0 : pluginOptions.host) || "", | ||
| sitemap: typeof (pluginOptions == null ? void 0 : pluginOptions.sitemap) === "undefined" ? true : pluginOptions.sitemap, | ||
| policy: (pluginOptions == null ? void 0 : pluginOptions.policy) || defaultOptions.policy | ||
| }; | ||
| return options; | ||
| }; | ||
| // src/utils/logger.ts | ||
| var Logger = class { | ||
| constructor(packageName2) { | ||
| this.colors = { | ||
| reset: "\x1B[0m", | ||
| fg: { | ||
| red: "\x1B[31m", | ||
| green: "\x1B[32m", | ||
| yellow: "\x1B[33m" | ||
| } | ||
| }; | ||
| this.packageName = packageName2; | ||
| } | ||
| log(msg, prefix = "") { | ||
| console.log(`%s${this.packageName}: ${msg}%s | ||
| `, prefix, prefix ? this.colors.reset : ""); | ||
| } | ||
| info(msg) { | ||
| this.log(msg); | ||
| } | ||
| success(msg) { | ||
| this.log(msg, this.colors.fg.green); | ||
| } | ||
| warn(msg) { | ||
| this.log(`Skipped! | ||
| ${msg}`, this.colors.fg.yellow); | ||
| } | ||
| error(msg) { | ||
| this.log(`Failed to create 'robots.txt'! | ||
| ${msg}`, this.colors.fg.red); | ||
| } | ||
| }; | ||
| // src/utils/is-valid-hostname.ts | ||
| var isValidHostname = (x) => { | ||
| if (typeof x !== "string") { | ||
| return false; | ||
| } | ||
| let value = x.toString(); | ||
| const validHostnameChars = /^[a-zA-Z0-9-.]{1,253}\.?$/g; | ||
@@ -26,10 +94,7 @@ if (!validHostnameChars.test(value)) { | ||
| // src/utils/isValidUrl.ts | ||
| // src/utils/is-valid-url.ts | ||
| var isValidUrl = (s) => { | ||
| if (typeof s !== "string") { | ||
| if (typeof s !== "string" || !s) { | ||
| return false; | ||
| } | ||
| if (!s) { | ||
| return false; | ||
| } | ||
| try { | ||
@@ -43,55 +108,27 @@ const dummy = new URL(s); | ||
| // src/get-robots-txt-content/helpers/addLine.ts | ||
| var capitaliseFirstLetter = (s) => s.charAt(0).toUpperCase() + s.slice(1); | ||
| var addLine = (name, rule) => { | ||
| if (rule && Array.isArray(rule) && rule.length > 0) { | ||
| let contents = ""; | ||
| rule.forEach((item) => { | ||
| contents += addLine(name, item); | ||
| }); | ||
| return contents; | ||
| // src/utils/is-valid-http-url.ts | ||
| var isValidHttpUrl = (s) => { | ||
| if (typeof s !== "string" || !s) { | ||
| return false; | ||
| } | ||
| const ruleContent = name === "Allow" || name === "Disallow" ? encodeURI(rule.toString()) : rule.toString(); | ||
| return `${capitaliseFirstLetter(name.replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase())}:${ruleContent.length > 0 ? ` ${ruleContent}` : ""} | ||
| `; | ||
| }; | ||
| var addLine_default = addLine; | ||
| // src/get-robots-txt-content/helpers/generatePoliceItem.ts | ||
| var generatePoliceItem = (item, index) => { | ||
| let contents = ""; | ||
| if (index !== 0) { | ||
| contents += "\n"; | ||
| try { | ||
| const { protocol } = new URL(s); | ||
| return protocol === "http:" || protocol === "https:"; | ||
| } catch { | ||
| return false; | ||
| } | ||
| contents += addLine_default("User-agent", item.userAgent); | ||
| if (typeof item.disallow === "string" || Array.isArray(item.disallow)) { | ||
| contents += addLine_default("Disallow", item.disallow); | ||
| } | ||
| if (item.allow) { | ||
| contents += addLine_default("Allow", item.allow); | ||
| } | ||
| if (item.crawlDelay) { | ||
| contents += addLine_default("Crawl-delay", item.crawlDelay); | ||
| } | ||
| if (item.cleanParam && item.cleanParam.length > 0) { | ||
| contents += addLine_default("Clean-param", item.cleanParam); | ||
| } | ||
| return contents; | ||
| }; | ||
| var generatePoliceItem_default = generatePoliceItem; | ||
| // src/get-robots-txt-content/index.ts | ||
| // src/is-opts-valid.ts | ||
| var logger; | ||
| var generateRobotsTxt = (policy, sitemap, host) => { | ||
| let contents = ""; | ||
| policy.forEach((item, index) => { | ||
| contents += generatePoliceItem_default(item, index); | ||
| }); | ||
| sitemap.forEach((item) => { | ||
| contents += addLine_default("Sitemap", item); | ||
| }); | ||
| if (host) { | ||
| contents += addLine_default("Host", host); | ||
| var validateSitemapItem = (sitemap) => { | ||
| if (!isValidUrl(sitemap)) { | ||
| logger.warn("Option `sitemap` contains not valid url."); | ||
| return false; | ||
| } | ||
| return contents; | ||
| if (!isValidHttpUrl(sitemap)) { | ||
| logger.warn("Option `sitemap` needs `http` or `https` protocol in url."); | ||
| return false; | ||
| } | ||
| return true; | ||
| }; | ||
@@ -103,15 +140,13 @@ var isValidSitemap = (sitemap) => { | ||
| } | ||
| if (!sitemap) { | ||
| return true; | ||
| } | ||
| if (typeof sitemap === "string") { | ||
| if (!isValidUrl(sitemap)) { | ||
| logger.warn("Option `sitemap` contains not valid url."); | ||
| return false; | ||
| } | ||
| } else if (Array.isArray(sitemap)) { | ||
| for (const item of sitemap) { | ||
| if (!isValidUrl(item)) { | ||
| if (sitemap) { | ||
| if (typeof sitemap === "string") { | ||
| if (!validateSitemapItem(sitemap)) { | ||
| return false; | ||
| } | ||
| } else if (Array.isArray(sitemap)) { | ||
| for (const item of sitemap) { | ||
| if (!validateSitemapItem(item)) { | ||
| return false; | ||
| } | ||
| } | ||
| } | ||
@@ -121,14 +156,2 @@ } | ||
| }; | ||
| var getSitemap = (sitemap, siteUrl) => { | ||
| if (!sitemap) { | ||
| return []; | ||
| } | ||
| if (typeof sitemap === "string") { | ||
| return [sitemap]; | ||
| } | ||
| if (Array.isArray(sitemap)) { | ||
| return sitemap; | ||
| } | ||
| return [`${siteUrl}${siteUrl.endsWith("/") ? "" : "/"}sitemap.xml`]; | ||
| }; | ||
| var isValidCleanParamItem = (item) => { | ||
@@ -186,69 +209,88 @@ if (typeof item !== "string") { | ||
| }; | ||
| var getRobotsTxtContent = (site, { host = "", sitemap = true, policy = [{ allow: "/", userAgent: "*" }] }, _logger) => { | ||
| var isOptsValid = (site, { host, sitemap, policy }, _logger) => { | ||
| logger = _logger; | ||
| if (!site) { | ||
| logger.warn("`site` property is required in `astro.config.mjs`."); | ||
| return void 0; | ||
| return false; | ||
| } | ||
| if (host && !isValidHostname(host)) { | ||
| logger.warn("Option `host` does not contain correct host."); | ||
| return void 0; | ||
| return false; | ||
| } | ||
| if (!isValidSitemap(sitemap)) { | ||
| return void 0; | ||
| if (sitemap && !isValidSitemap(sitemap)) { | ||
| return false; | ||
| } | ||
| const siteMap = getSitemap(sitemap, site); | ||
| if (!isValidPolicy(policy)) { | ||
| return void 0; | ||
| if (policy && !isValidPolicy(policy)) { | ||
| return false; | ||
| } | ||
| const robotsTxtContent = generateRobotsTxt(policy, siteMap, host); | ||
| return robotsTxtContent; | ||
| return true; | ||
| }; | ||
| // src/utils/logger.ts | ||
| var Logger = class { | ||
| constructor(packageName2) { | ||
| this.colors = { | ||
| reset: "\x1B[0m", | ||
| fg: { | ||
| red: "\x1B[31m", | ||
| green: "\x1B[32m", | ||
| yellow: "\x1B[33m" | ||
| } | ||
| }; | ||
| this.packageName = packageName2; | ||
| // src/get-robots-txt-content.ts | ||
| var capitaliseFirstLetter = (s) => s.charAt(0).toUpperCase() + s.slice(1); | ||
| var addBackslash = (s) => s.endsWith("/") ? s : `${s}/`; | ||
| var addLine = (name, rule) => { | ||
| if (rule && Array.isArray(rule) && rule.length > 0) { | ||
| let contents = ""; | ||
| rule.forEach((item) => { | ||
| contents += addLine(name, item); | ||
| }); | ||
| return contents; | ||
| } | ||
| log(msg, prefix = "") { | ||
| console.log(`%s${this.packageName}: ${msg}%s | ||
| `, prefix, prefix ? this.colors.reset : ""); | ||
| const ruleContent = name === "Allow" || name === "Disallow" ? encodeURI(rule.toString()) : rule.toString(); | ||
| return `${capitaliseFirstLetter(name.replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase())}:${ruleContent.length > 0 ? ` ${ruleContent}` : ""} | ||
| `; | ||
| }; | ||
| var generatePoliceItem = (item, index) => { | ||
| let contents = ""; | ||
| if (index !== 0) { | ||
| contents += "\n"; | ||
| } | ||
| info(msg) { | ||
| this.log(msg); | ||
| contents += addLine("User-agent", item.userAgent); | ||
| if (typeof item.disallow === "string" || Array.isArray(item.disallow)) { | ||
| contents += addLine("Disallow", item.disallow); | ||
| } | ||
| success(msg) { | ||
| this.log(msg, this.colors.fg.green); | ||
| if (item.allow) { | ||
| contents += addLine("Allow", item.allow); | ||
| } | ||
| warn(msg) { | ||
| this.log(`Skipped! | ||
| ${msg}`, this.colors.fg.yellow); | ||
| if (item.crawlDelay) { | ||
| contents += addLine("Crawl-delay", item.crawlDelay); | ||
| } | ||
| error(msg) { | ||
| this.log(`Failed to create 'robots.txt'! | ||
| ${msg}`, this.colors.fg.red); | ||
| if (item.cleanParam && item.cleanParam.length > 0) { | ||
| contents += addLine("Clean-param", item.cleanParam); | ||
| } | ||
| return contents; | ||
| }; | ||
| var getSitemapArr = (sitemap, site) => { | ||
| if (typeof sitemap === "boolean" && !sitemap) { | ||
| return []; | ||
| } | ||
| if (!sitemap) { | ||
| return []; | ||
| } | ||
| if (typeof sitemap === "string") { | ||
| return [sitemap]; | ||
| } | ||
| if (Array.isArray(sitemap)) { | ||
| return sitemap; | ||
| } | ||
| return [`${addBackslash(site)}sitemap.xml`]; | ||
| }; | ||
| var getRobotsTxtContent = (site, { host, sitemap, policy }) => { | ||
| let result = ""; | ||
| policy == null ? void 0 : policy.forEach((item, index) => { | ||
| result += generatePoliceItem(item, index); | ||
| }); | ||
| getSitemapArr(sitemap, site).forEach((item) => { | ||
| result += addLine("Sitemap", item); | ||
| }); | ||
| if (host) { | ||
| result += addLine("Host", host); | ||
| } | ||
| return result; | ||
| }; | ||
| // src/index.ts | ||
| var logger2 = new Logger(packageName); | ||
| var defaultOptions = { | ||
| host: "", | ||
| sitemap: true, | ||
| policy: [ | ||
| { | ||
| allow: "/", | ||
| userAgent: "*" | ||
| } | ||
| ] | ||
| }; | ||
| var createPlugin = (pluginOptions = defaultOptions) => { | ||
| var createPlugin = (pluginOptions = {}) => { | ||
| let config; | ||
@@ -258,10 +300,11 @@ return { | ||
| hooks: { | ||
| "astro:config:done": async ({ config: _config }) => { | ||
| config = _config; | ||
| "astro:config:done": async ({ config: cfg }) => { | ||
| config = cfg; | ||
| }, | ||
| "astro:build:done": async ({ dir }) => { | ||
| const robotsTxtContent = getRobotsTxtContent(config.site, pluginOptions, logger2); | ||
| if (!robotsTxtContent) { | ||
| const opts = withOptions(pluginOptions); | ||
| if (!isOptsValid(config.site, opts, logger2)) { | ||
| return; | ||
| } | ||
| const robotsTxtContent = getRobotsTxtContent(config.site, opts); | ||
| try { | ||
@@ -268,0 +311,0 @@ const url = new URL("robots.txt", dir); |
| { | ||
| "name": "astro-robots-txt", | ||
| "version": "0.1.14", | ||
| "version": "0.1.15", | ||
| "description": "Generate a robots.txt for Astro", | ||
@@ -40,7 +40,7 @@ "keywords": [ | ||
| "@types/node": "^17.0.31", | ||
| "@typescript-eslint/eslint-plugin": "^5.22.0", | ||
| "@typescript-eslint/parser": "^5.22.0", | ||
| "astro": "^1.0.0-beta.24", | ||
| "@typescript-eslint/eslint-plugin": "^5.23.0", | ||
| "@typescript-eslint/parser": "^5.23.0", | ||
| "astro": "^1.0.0-beta.27", | ||
| "esbuild": "^0.14.38", | ||
| "eslint": "^8.14.0", | ||
| "eslint": "^8.15.0", | ||
| "eslint-config-airbnb-base": "^15.0.0", | ||
@@ -52,5 +52,5 @@ "eslint-config-airbnb-typescript": "^17.0.0", | ||
| "eslint-plugin-prettier": "^4.0.0", | ||
| "jest": "28.0.3", | ||
| "jest": "28.1.0", | ||
| "prettier": "^2.6.2", | ||
| "ts-jest": "^28.0.1", | ||
| "ts-jest": "^28.0.2", | ||
| "typescript": "^4.6.4" | ||
@@ -65,8 +65,8 @@ }, | ||
| "preview": "astro --root demo preview", | ||
| "i:all": "pnpm recursive install", | ||
| "format": "prettier -w .", | ||
| "lint": "eslint . --ext .ts", | ||
| "lint:fix": "eslint . --fix --ext .ts", | ||
| "test": "jest" | ||
| "test": "jest", | ||
| "test:pub": "pnpm publish --dry-run --no-git-checks" | ||
| } | ||
| } |
@@ -11,3 +11,4 @@ <a href="https://war.ukraine.ua/support-ukraine/"> | ||
| ------ | ||
| --- | ||
| The _robots.txt_ file informs search engines which pages on your website should be crawled. [See Google's own advice on robots.txt](https://developers.google.com/search/docs/advanced/robots/intro) to learn more. | ||
@@ -18,4 +19,4 @@ | ||
| For Astro project you usually create the _robots.txt_ in a text editor and place it to the `public/` directory. | ||
| In that case you must manually synchronize `site` option in _astro.config.*_ with `Sitemap:` record in _robots.txt_. | ||
| It brakes DRY principle. | ||
| In that case you must manually synchronize `site` option in _astro.config.\*_ with `Sitemap:` record in _robots.txt_. | ||
| It brakes DRY principle. | ||
@@ -27,3 +28,3 @@ Sometimes, especially during development, it's needed to prevent your site from being indexed. To achieve this you need place meta tag `<meta name="robots" content="noindex">` in the `<head>` section of pages or add `X-Robots-Tag: noindex` in HTTP header response, then add lines `User-agent: *` and `Disallow: \` to _robots.txt_. | ||
| ------ | ||
| --- | ||
@@ -36,3 +37,3 @@ ## Installation | ||
| You should run `astro add` command in your project directory. This command after prompt will install required dependencies and apply changes to _astro.config.*_. | ||
| You should run `astro add` command in your project directory. This command after prompt will install required dependencies and apply changes to _astro.config.\*_. | ||
@@ -67,11 +68,11 @@ ```sh | ||
| Then apply this integration to your _astro.config.*_. All details below in **Getting started**. | ||
| Then apply this integration to your _astro.config.\*_. All details below in **Getting started**. | ||
| ## Getting started | ||
| The `astro-robots-txt` integration requires a deployment / site URL for generation. Add your site's URL under your _astro.config.*_ using the `site` property. | ||
| The `astro-robots-txt` integration requires a deployment / site URL for generation. Add your site's URL under your _astro.config.\*_ using the `site` property. | ||
| :exclamation: Provide the `experimental` property to your _astro.config.*_, because only official **@astrojs/\*** integrations are currently supported by Astro. Set the `experimental.integrations` value to `true`. | ||
| :exclamation: Provide the `experimental` property to your _astro.config.\*_, because only official **@astrojs/\*** integrations are currently supported by Astro. Set the `experimental.integrations` value to `true`. | ||
| Then, apply this integration to your _astro.config.*_ file using the `integrations` property. | ||
| Then, apply this integration to your _astro.config.\*_ file using the `integrations` property. | ||
@@ -89,3 +90,3 @@ **astro.config.mjs** | ||
| // Important! | ||
| // Only official '@astrojs/*' integrations are currently supported by Astro. | ||
| // Only official '@astrojs/*' integrations are currently supported by Astro. | ||
| // Add 'experimental.integrations: true' to make 'astro-robots-txt' working | ||
@@ -95,3 +96,3 @@ // with 'astro build' command. | ||
| integrations: true, | ||
| }, | ||
| }, | ||
| integrations: [robotsTxt()], | ||
@@ -117,9 +118,9 @@ }); | ||
| | Name | Type | Default | Description | | ||
| | :-------: | :-----------------------------: | :------------------------------: | :------------------------------------------------------------------: | | ||
| | `host` | `String` | `` | Host of your site | | ||
| | `sitemap` | `Boolean / String` / `String[]` | `true` | Resulting output in a _robots.txt_ will be `Sitemap: your-site-url/sitemap.xml` | | ||
| | | | | If `sitemap: false` - no `Sitemap` line in the output. | | ||
| | | | | You could use for the `sitemap` a valid url string or array of url strings. | | ||
| | `policy` | `Policy[]` | [{ allow: '/', userAgent: '*' }] | List of `Policy` rules | | ||
| | Name | Type | Default | Description | | ||
| | :-------: | :-----------------------------: | :------------------------------: | :-------------------------------------------------------------------------------------------: | | ||
| | `host` | `String` | `` | Host of your site | | ||
| | `sitemap` | `Boolean / String` / `String[]` | `true` | Resulting output in a _robots.txt_ will be `Sitemap: your-site-url/sitemap.xml` | | ||
| | | | | If `sitemap: false` - no `Sitemap` line in the output. | | ||
| | | | | You could use for the `sitemap` a valid **http** url string or array of **http** url strings. | | ||
| | `policy` | `Policy[]` | [{ allow: '/', userAgent: '*' }] | List of `Policy` rules | | ||
@@ -147,10 +148,7 @@ ### Policy | ||
| integrations: true, | ||
| }, | ||
| }, | ||
| integrations: [ | ||
| robotsTxt({ | ||
| host: 'example.com', | ||
| sitemap: [ | ||
| 'https://example.com/main-sitemap.xml', | ||
| 'https://example.com/images-sitemap.xml' | ||
| ], | ||
| sitemap: ['https://example.com/main-sitemap.xml', 'https://example.com/images-sitemap.xml'], | ||
| policy: [ | ||
@@ -194,3 +192,3 @@ { | ||
| integrations: true, | ||
| }, | ||
| }, | ||
| integrations: [ | ||
@@ -207,7 +205,7 @@ robotsTxt({ | ||
| Only official **@astrojs/\*** integrations are currently supported by Astro. | ||
| Only official **@astrojs/\*** integrations are currently supported by Astro. | ||
| There are two possibilities to make **astro-robots-txt** integration working with current version of Astro. | ||
| There are two possibilities to make **astro-robots-txt** integration working with current version of Astro. | ||
| Set the `experimental.integrations` option to `true` in your _astro.config.*_. | ||
| Set the `experimental.integrations` option to `true` in your _astro.config.\*_. | ||
@@ -220,7 +218,7 @@ ```js | ||
| integrations: true, | ||
| }, | ||
| }, | ||
| }); | ||
| ``` | ||
| Or use the `--experimental-integrations` flag for build command. | ||
| Or use the `--experimental-integrations` flag for build command. | ||
@@ -227,0 +225,0 @@ ```sh |
Fixed alerts
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Improved metrics
- Total package byte prevSize
- increased by6.02%
19990
- Lines of code
- increased by14.59%
322
- Number of medium supply chain risk alerts
- decreased by-100%
0
Worsened metrics
- Number of lines in readme file
- decreased by-0.88%
224
No dependency changes