third-party-web
Advanced tools
Comparing version 0.4.0 to 0.5.0
@@ -1,49 +0,3 @@ | ||
const entities = require('../data/entities.json') | ||
const DOMAIN_IN_URL_REGEX = /:\/\/(.*?)(\/|$)/ | ||
const DOMAIN_CHARACTERS = /[a-z0-9.-]+\.[a-z0-9]+/i | ||
const ROOT_DOMAIN_REGEX = /[^.]+\.([^.]+|(gov|com|co|ne)\.\w{2})$/i | ||
const entityByDomain = new Map() | ||
const entityByRootDomain = new Map() | ||
for (const entity of entities) { | ||
if (!entity.company) entity.company = entity.name | ||
for (const domain of entity.domains) { | ||
if (entityByDomain.has(domain)) throw new Error(`Duplicate domain ${domain}`) | ||
entityByDomain.set(domain, entity) | ||
const rootDomain = getRootDomain(domain) | ||
if (entityByRootDomain.has(rootDomain) && entityByRootDomain.get(rootDomain) !== entity) { | ||
entityByRootDomain.set(rootDomain, false) | ||
} else { | ||
entityByRootDomain.set(rootDomain, entity) | ||
} | ||
} | ||
} | ||
for (const [rootDomain, entity] of entityByRootDomain.entries()) { | ||
if (!entity) entityByRootDomain.delete(rootDomain) | ||
} | ||
function getDomainFromOriginOrURL(originOrURL) { | ||
if (DOMAIN_IN_URL_REGEX.test(originOrURL)) return originOrURL.match(DOMAIN_IN_URL_REGEX)[1] | ||
if (DOMAIN_CHARACTERS.test(originOrURL)) return originOrURL.match(DOMAIN_CHARACTERS)[0] | ||
throw new Error(`Unable to find domain in "${originOrURL}"`) | ||
} | ||
function getRootDomain(originOrURL) { | ||
const domain = getDomainFromOriginOrURL(originOrURL) | ||
const match = domain.match(ROOT_DOMAIN_REGEX) | ||
return (match && match[0]) || domain | ||
} | ||
function getEntity(originOrURL) { | ||
const domain = getDomainFromOriginOrURL(originOrURL) | ||
const rootDomain = getRootDomain(domain) | ||
if (entityByDomain.has(domain)) return entityByDomain.get(domain) | ||
if (entityByRootDomain.has(rootDomain)) return entityByRootDomain.get(rootDomain) | ||
return undefined | ||
} | ||
module.exports = {entities, getRootDomain, getEntity} | ||
const {createAPIFromDataset} = require('./create-entity-finder-api.js') | ||
const entities = require('../dist/entities.json') | ||
module.exports = createAPIFromDataset(entities) |
@@ -1,2 +0,5 @@ | ||
const {getRootDomain, getEntity} = require('./index.js') | ||
const fs = require('fs') | ||
const path = require('path') | ||
const JSON5 = require('json5') | ||
const {entities, getRootDomain, getEntity} = require('./index.js') | ||
@@ -46,1 +49,19 @@ describe('getRootDomain', () => { | ||
}) | ||
describe('build state', () => { | ||
it('should use the complete entities set', () => { | ||
const json = fs.readFileSync(path.join(__dirname, '../data/entities.json5'), 'utf8') | ||
const sourceOfTruthEntities = JSON5.parse(json) | ||
expect(entities).toHaveLength(sourceOfTruthEntities.length) | ||
}) | ||
it('should have all the same subsets in root as lib', () => { | ||
const srcSizes = fs.readdirSync(path.join(__dirname, 'subsets')) | ||
const dstSizes = fs.readdirSync(path.join(__dirname, '../')).filter(f => f.includes('-subset')) | ||
expect(dstSizes).toHaveLength(srcSizes.length) // run `yarn build` if this fails | ||
for (const file of dstSizes) { | ||
if (file.endsWith('.js')) require(path.join(__dirname, '../', file)) | ||
} | ||
}) | ||
}) |
{ | ||
"name": "third-party-web", | ||
"version": "0.4.0", | ||
"version": "0.5.0", | ||
"description": "Categorized data on third party entities on the web.", | ||
"main": "./lib/index.js", | ||
"scripts": { | ||
"build": "node bin/build-entity-json-files.js && generate-export-aliases", | ||
"fix-entities": "prettier --print-width=100 --write data/entities.json", | ||
"start": "mkdir -p .tmp && node bin/merge-origins-with-entities.js && node bin/create-markdown.js", | ||
"test": "jest" | ||
"test": "npm run build && jest" | ||
}, | ||
@@ -25,6 +26,13 @@ "keywords": [ | ||
"colors": "^1.3.3", | ||
"generate-export-aliases": "^1.1.0", | ||
"jest": "^24.5.0", | ||
"json5": "^2.1.0", | ||
"lodash": "^4.17.10", | ||
"prettier": "^1.13.7" | ||
}, | ||
"config": { | ||
"exportAliases": { | ||
"httparchive-subset": "./lib/subsets/httparchive.js" | ||
} | ||
} | ||
} |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Dynamic require
Supply chain riskDynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.
Found 1 instance in 1 package
Filesystem access
Supply chain riskAccesses the file system, and could potentially read sensitive data.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
346367
16
8
120
2
1