Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

daq-proc

Package Overview
Dependencies
Maintainers
1
Versions
22
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

daq-proc - npm Package Compare versions

Comparing version 2.1.2 to 3.0.0

dist/daq-proc.2.1.3.js
dist/daq-proc.3.0.0.js

2

demo/daq-proc-app.js

@@ -30,3 +30,3 @@ // exposing the underlying libraries in a transparent way

// Prep for defining arrays of language to remove stopwords from.
const languages = { af: sw.af, bn: sw.bn, br: sw.br, zh: sw.zh, da: sw.da, nl: sw.nl, en: sw.en, fa: sw.fa, fi: sw.fi, fr: sw.fr, de: sw.de, ha: sw.ha, he: sw.he, hi: sw.hi, id: sw.id, it: sw.it, ja: sw.ja, lgg: sw.lgg, lggo: sw.lggo, ar: sw.ar, no: sw.no, pl: sw.pl, pt: sw.pt, pa: sw.pa, ru: sw.ru, so: sw.so, st: sw.st, es: sw.es, sw: sw.sw, sv: sw.sv, vi: sw.vi, yo: sw.yo, zu: sw.zu }
const languages = { af: sw.af, ar: sw.ar, bg: sw.bg, bn: sw.bn, br: sw.br, ca: sw.ca, cs: sw.cs, da: sw.da, de: sw.de, el: sw.el, en: sw.en, eo: sw.eo, es: sw.es, et: sw.et, eu: sw.eu, fa: sw.fa, fi: sw.fi, fr: sw.fr, ga: sw.ga, gl: sw.gl, ha: sw.ha, he: sw.he, hi: sw.hi, hr: sw.hr, hu: sw.hu, hy: sw.hy, id: sw.id, it: sw.it, ja: sw.ja, ko: sw.ko, la: sw.la, lgg: sw.lgg, lggo: sw.lggo, lv: sw.lv, mr: sw.mr, nl: sw.nl, no: sw.no, pl: sw.pl, pt: sw.pt, ptbr: sw.ptbr, pa: sw.pa, ro: sw.ro, ru: sw.ru, sk: sw.sk, sl: sw.sl, so: sw.so, st: sw.st, sv: sw.sv, sw: sw.sw, th: sw.th, tr: sw.tr, vi: sw.vi, yo: sw.yo, zh: sw.zh, zu: sw.zu }
let code = document.getElementById("languageSelected").value || en

@@ -33,0 +33,0 @@ console.log(code)

16

package.json
{
"name": "daq-proc",
"version": "2.1.2",
"description": "Simple document processor for [nowsearch.xyz](https://github.com/eklem/nowsearch.xyz) to makes search running in the browser and node.js a little better. Removes stopwords (smaller index and less irrelevant hits), extract keywords to filter on and prepares ngrams for auto-complete functionality.",
"version": "3.0.0",
"description": "Simple document processor to make search running in the browser and node.js a little better. Supports 50+ languages. Removes stopwords (smaller index and less irrelevant hits), extract keywords to filter on and prepares ngrams for auto-complete functionality.",
"main": "index.js",

@@ -22,14 +22,14 @@ "scripts": {

"words-n-numbers": "^2.0.1",
"stopword": "^0.3.5",
"stopword": "^1.0.0",
"eklem-headline-parser": "^2.0.4",
"ngraminator": "^2.0.1",
"hit-highlighter": "1.0.1",
"hit-highlighter": "2.0.0",
"cheerio": "1.0.0-rc.3"
},
"devDependencies": {
"standard": "^14.3.2",
"webpack": "^4.42.0",
"standard": "^14.3.3",
"webpack": "^4.42.1",
"webpack-cli": "^3.3.11",
"tape": "^4.13.2",
"tape-run": "^6.0.1"
"tape": "^5.0.0-next.5",
"tape-run": "^7.0.0"
},

@@ -36,0 +36,0 @@ "author": "Espen Klem",

8

README.md

@@ -84,9 +84,3 @@ # daq-proc

// returns:
//[
// 'some',
// '<span class="highlighted">interesting</span>',
// '<span class="highlighted">words</span>',
// 'to',
// 'remember'
//]
// 'some <span class="highlighted">interesting words</span> to remember'
</script>

@@ -93,0 +87,0 @@ ```

20

test/test.js

@@ -10,35 +10,35 @@ const test = require('tape')

const headlineString = $('h1').text()
t.looseEqual(headlineString, 'Saudi Arabia vows to respond to oil attacks with \'necessary measures\'')
t.deepEqual(headlineString, 'Saudi Arabia vows to respond to oil attacks with \'necessary measures\'')
// Extracting the first 2501 characters from bodytext / story, only paragraphs
const bodyString = $('.story-body__inner p').text().substring(0, 2501)
t.looseEqual(bodyString, 'Saudi Arabia says it will respond with "necessary measures" to attacks on two oil facilities as it reiterated the accusation that Iran was behind them.Minister of State for Foreign Affairs Adel al-Jubeir said the weapons used were Iranian and vowed to release the full findings of the investigation.Iran denies involvement in the attacks.Earlier, a senior Iranian military official said Iran was ready to destroy any aggressor after the US announced it was sending troops to Saudi Arabia.Iranian-backed Houthi rebels in Yemen have said they were responsible for the drone and missile strikes on 14 September that affected the global oil supply.Tensions between the US and Iran have escalated since US President Donald Trump abandoned a deal limiting Iran\'s nuclear activities last year and reinstated sanctions.Speaking to reporters in Riyadh, Mr Jubeir said Saudi Arabia was in consultation with its allies and would take necessary and suitable measures after its investigation was complete, without giving details of possible actions. He repeated that the strikes targeting the Abqaiq oil facility and the Khurais oil field had come from the north and not from Yemen but did not give a specific location, and urged the international community to take a stand."The kingdom calls upon the international community to assume its responsibility in condemning those that stand behind this act, and to take a firm and clear position against this reckless behaviour that threatens the global economy," he said.The Saudi defence ministry showed off on Wednesday what it said were the remains of drones and cruise missiles proving Iranian involvement.The US has also accused Iran of being behind the attacks, and unnamed senior officials have told US media that the evidence suggests the strikes originated in the south of Iran.On Friday, Secretary of Defence Mark Esper said the US would send a yet-to-be-decided number of troops to Saudi Arabia to boost the country\'s air and missile defences.President Trump then announced new sanctions against Iran, focusing on the country\'s central bank and its sovereign wealth fund, while signalling that he wanted to avoid military conflict.The news conference in the glittering halls of Saudi Arabia\'s foreign ministry was fairly predictable in its customary condemnation of Iran.The Islamic Republic, said the eloquent and softly-spoken minister of state for foreign affairs, was to blame for all the mischief-making in the region, including more than 260 ballistic')
t.deepEqual(bodyString, 'Saudi Arabia says it will respond with "necessary measures" to attacks on two oil facilities as it reiterated the accusation that Iran was behind them.Minister of State for Foreign Affairs Adel al-Jubeir said the weapons used were Iranian and vowed to release the full findings of the investigation.Iran denies involvement in the attacks.Earlier, a senior Iranian military official said Iran was ready to destroy any aggressor after the US announced it was sending troops to Saudi Arabia.Iranian-backed Houthi rebels in Yemen have said they were responsible for the drone and missile strikes on 14 September that affected the global oil supply.Tensions between the US and Iran have escalated since US President Donald Trump abandoned a deal limiting Iran\'s nuclear activities last year and reinstated sanctions.Speaking to reporters in Riyadh, Mr Jubeir said Saudi Arabia was in consultation with its allies and would take necessary and suitable measures after its investigation was complete, without giving details of possible actions. He repeated that the strikes targeting the Abqaiq oil facility and the Khurais oil field had come from the north and not from Yemen but did not give a specific location, and urged the international community to take a stand."The kingdom calls upon the international community to assume its responsibility in condemning those that stand behind this act, and to take a firm and clear position against this reckless behaviour that threatens the global economy," he said.The Saudi defence ministry showed off on Wednesday what it said were the remains of drones and cruise missiles proving Iranian involvement.The US has also accused Iran of being behind the attacks, and unnamed senior officials have told US media that the evidence suggests the strikes originated in the south of Iran.On Friday, Secretary of Defence Mark Esper said the US would send a yet-to-be-decided number of troops to Saudi Arabia to boost the country\'s air and missile defences.President Trump then announced new sanctions against Iran, focusing on the country\'s central bank and its sovereign wealth fund, while signalling that he wanted to avoid military conflict.The news conference in the glittering halls of Saudi Arabia\'s foreign ministry was fairly predictable in its customary condemnation of Iran.The Islamic Republic, said the eloquent and softly-spoken minister of state for foreign affairs, was to blame for all the mischief-making in the region, including more than 260 ballistic')
// Extract array of words from title / headline
const headlineArray = wnn.extract(headlineString, { regex: wnn.wordsAndNumbers, toLowercase: true })
t.looseEqual(headlineArray, ['saudi', 'arabia', 'vows', 'to', 'respond', 'to', 'oil', 'attacks', 'with', 'necessary', 'measures'])
t.deepEqual(headlineArray, ['saudi', 'arabia', 'vows', 'to', 'respond', 'to', 'oil', 'attacks', 'with', 'necessary', 'measures'])
// Extract array of words from bodytext / story
const bodyArray = wnn.extract(bodyString, { regex: wnn.wordsAndNumbers, toLowercase: true })
t.looseEqual(bodyArray, ['saudi', 'arabia', 'says', 'it', 'will', 'respond', 'with', 'necessary', 'measures', 'to', 'attacks', 'on', 'two', 'oil', 'facilities', 'as', 'it', 'reiterated', 'the', 'accusation', 'that', 'iran', 'was', 'behind', 'them', 'minister', 'of', 'state', 'for', 'foreign', 'affairs', 'adel', 'al', 'jubeir', 'said', 'the', 'weapons', 'used', 'were', 'iranian', 'and', 'vowed', 'to', 'release', 'the', 'full', 'findings', 'of', 'the', 'investigation', 'iran', 'denies', 'involvement', 'in', 'the', 'attacks', 'earlier', 'a', 'senior', 'iranian', 'military', 'official', 'said', 'iran', 'was', 'ready', 'to', 'destroy', 'any', 'aggressor', 'after', 'the', 'us', 'announced', 'it', 'was', 'sending', 'troops', 'to', 'saudi', 'arabia', 'iranian', 'backed', 'houthi', 'rebels', 'in', 'yemen', 'have', 'said', 'they', 'were', 'responsible', 'for', 'the', 'drone', 'and', 'missile', 'strikes', 'on', '14', 'september', 'that', 'affected', 'the', 'global', 'oil', 'supply', 'tensions', 'between', 'the', 'us', 'and', 'iran', 'have', 'escalated', 'since', 'us', 'president', 'donald', 'trump', 'abandoned', 'a', 'deal', 'limiting', 'iran', 's', 'nuclear', 'activities', 'last', 'year', 'and', 'reinstated', 'sanctions', 'speaking', 'to', 'reporters', 'in', 'riyadh', 'mr', 'jubeir', 'said', 'saudi', 'arabia', 'was', 'in', 'consultation', 'with', 'its', 'allies', 'and', 'would', 'take', 'necessary', 'and', 'suitable', 'measures', 'after', 'its', 'investigation', 'was', 'complete', 'without', 'giving', 'details', 'of', 'possible', 'actions', 'he', 'repeated', 'that', 'the', 'strikes', 'targeting', 'the', 'abqaiq', 'oil', 'facility', 'and', 'the', 'khurais', 'oil', 'field', 'had', 'come', 'from', 'the', 'north', 'and', 'not', 'from', 'yemen', 'but', 'did', 'not', 'give', 'a', 'specific', 'location', 'and', 'urged', 'the', 'international', 'community', 'to', 'take', 'a', 'stand', 'the', 'kingdom', 'calls', 'upon', 'the', 'international', 'community', 'to', 'assume', 'its', 'responsibility', 'in', 'condemning', 'those', 'that', 'stand', 'behind', 'this', 'act', 'and', 'to', 'take', 'a', 'firm', 'and', 'clear', 'position', 'against', 'this', 'reckless', 'behaviour', 'that', 'threatens', 'the', 'global', 'economy', 'he', 'said', 'the', 'saudi', 'defence', 'ministry', 'showed', 'off', 'on', 'wednesday', 'what', 'it', 'said', 'were', 'the', 'remains', 'of', 'drones', 'and', 'cruise', 'missiles', 'proving', 'iranian', 'involvement', 'the', 'us', 'has', 'also', 'accused', 'iran', 'of', 'being', 'behind', 'the', 'attacks', 'and', 'unnamed', 'senior', 'officials', 'have', 'told', 'us', 'media', 'that', 'the', 'evidence', 'suggests', 'the', 'strikes', 'originated', 'in', 'the', 'south', 'of', 'iran', 'on', 'friday', 'secretary', 'of', 'defence', 'mark', 'esper', 'said', 'the', 'us', 'would', 'send', 'a', 'yet', 'to', 'be', 'decided', 'number', 'of', 'troops', 'to', 'saudi', 'arabia', 'to', 'boost', 'the', 'country', 's', 'air', 'and', 'missile', 'defences', 'president', 'trump', 'then', 'announced', 'new', 'sanctions', 'against', 'iran', 'focusing', 'on', 'the', 'country', 's', 'central', 'bank', 'and', 'its', 'sovereign', 'wealth', 'fund', 'while', 'signalling', 'that', 'he', 'wanted', 'to', 'avoid', 'military', 'conflict', 'the', 'news', 'conference', 'in', 'the', 'glittering', 'halls', 'of', 'saudi', 'arabia', 's', 'foreign', 'ministry', 'was', 'fairly', 'predictable', 'in', 'its', 'customary', 'condemnation', 'of', 'iran', 'the', 'islamic', 'republic', 'said', 'the', 'eloquent', 'and', 'softly', 'spoken', 'minister', 'of', 'state', 'for', 'foreign', 'affairs', 'was', 'to', 'blame', 'for', 'all', 'the', 'mischief', 'making', 'in', 'the', 'region', 'including', 'more', 'than', '260', 'ballistic'])
t.deepEqual(bodyArray, ['saudi', 'arabia', 'says', 'it', 'will', 'respond', 'with', 'necessary', 'measures', 'to', 'attacks', 'on', 'two', 'oil', 'facilities', 'as', 'it', 'reiterated', 'the', 'accusation', 'that', 'iran', 'was', 'behind', 'them', 'minister', 'of', 'state', 'for', 'foreign', 'affairs', 'adel', 'al', 'jubeir', 'said', 'the', 'weapons', 'used', 'were', 'iranian', 'and', 'vowed', 'to', 'release', 'the', 'full', 'findings', 'of', 'the', 'investigation', 'iran', 'denies', 'involvement', 'in', 'the', 'attacks', 'earlier', 'a', 'senior', 'iranian', 'military', 'official', 'said', 'iran', 'was', 'ready', 'to', 'destroy', 'any', 'aggressor', 'after', 'the', 'us', 'announced', 'it', 'was', 'sending', 'troops', 'to', 'saudi', 'arabia', 'iranian', 'backed', 'houthi', 'rebels', 'in', 'yemen', 'have', 'said', 'they', 'were', 'responsible', 'for', 'the', 'drone', 'and', 'missile', 'strikes', 'on', '14', 'september', 'that', 'affected', 'the', 'global', 'oil', 'supply', 'tensions', 'between', 'the', 'us', 'and', 'iran', 'have', 'escalated', 'since', 'us', 'president', 'donald', 'trump', 'abandoned', 'a', 'deal', 'limiting', 'iran', 's', 'nuclear', 'activities', 'last', 'year', 'and', 'reinstated', 'sanctions', 'speaking', 'to', 'reporters', 'in', 'riyadh', 'mr', 'jubeir', 'said', 'saudi', 'arabia', 'was', 'in', 'consultation', 'with', 'its', 'allies', 'and', 'would', 'take', 'necessary', 'and', 'suitable', 'measures', 'after', 'its', 'investigation', 'was', 'complete', 'without', 'giving', 'details', 'of', 'possible', 'actions', 'he', 'repeated', 'that', 'the', 'strikes', 'targeting', 'the', 'abqaiq', 'oil', 'facility', 'and', 'the', 'khurais', 'oil', 'field', 'had', 'come', 'from', 'the', 'north', 'and', 'not', 'from', 'yemen', 'but', 'did', 'not', 'give', 'a', 'specific', 'location', 'and', 'urged', 'the', 'international', 'community', 'to', 'take', 'a', 'stand', 'the', 'kingdom', 'calls', 'upon', 'the', 'international', 'community', 'to', 'assume', 'its', 'responsibility', 'in', 'condemning', 'those', 'that', 'stand', 'behind', 'this', 'act', 'and', 'to', 'take', 'a', 'firm', 'and', 'clear', 'position', 'against', 'this', 'reckless', 'behaviour', 'that', 'threatens', 'the', 'global', 'economy', 'he', 'said', 'the', 'saudi', 'defence', 'ministry', 'showed', 'off', 'on', 'wednesday', 'what', 'it', 'said', 'were', 'the', 'remains', 'of', 'drones', 'and', 'cruise', 'missiles', 'proving', 'iranian', 'involvement', 'the', 'us', 'has', 'also', 'accused', 'iran', 'of', 'being', 'behind', 'the', 'attacks', 'and', 'unnamed', 'senior', 'officials', 'have', 'told', 'us', 'media', 'that', 'the', 'evidence', 'suggests', 'the', 'strikes', 'originated', 'in', 'the', 'south', 'of', 'iran', 'on', 'friday', 'secretary', 'of', 'defence', 'mark', 'esper', 'said', 'the', 'us', 'would', 'send', 'a', 'yet', 'to', 'be', 'decided', 'number', 'of', 'troops', 'to', 'saudi', 'arabia', 'to', 'boost', 'the', 'country', 's', 'air', 'and', 'missile', 'defences', 'president', 'trump', 'then', 'announced', 'new', 'sanctions', 'against', 'iran', 'focusing', 'on', 'the', 'country', 's', 'central', 'bank', 'and', 'its', 'sovereign', 'wealth', 'fund', 'while', 'signalling', 'that', 'he', 'wanted', 'to', 'avoid', 'military', 'conflict', 'the', 'news', 'conference', 'in', 'the', 'glittering', 'halls', 'of', 'saudi', 'arabia', 's', 'foreign', 'ministry', 'was', 'fairly', 'predictable', 'in', 'its', 'customary', 'condemnation', 'of', 'iran', 'the', 'islamic', 'republic', 'said', 'the', 'eloquent', 'and', 'softly', 'spoken', 'minister', 'of', 'state', 'for', 'foreign', 'affairs', 'was', 'to', 'blame', 'for', 'all', 'the', 'mischief', 'making', 'in', 'the', 'region', 'including', 'more', 'than', '260', 'ballistic'])
// Remove stopwords from title / headline
const headlineStopped = sw.removeStopwords(headlineArray)
t.looseEqual(headlineStopped, ['saudi', 'arabia', 'vows', 'respond', 'oil', 'attacks', 'necessary', 'measures'])
t.deepEqual(headlineStopped, ['saudi', 'arabia', 'vows', 'respond', 'oil', 'attacks', 'necessary', 'measures'])
// Remove stopwords from bodytext / story
const bodyStopped = sw.removeStopwords(bodyArray)
t.looseEqual(bodyStopped, ['saudi', 'arabia', 'says', 'will', 'respond', 'necessary', 'measures', 'attacks', 'two', 'oil', 'facilities', 'reiterated', 'accusation', 'iran', 'behind', 'minister', 'state', 'foreign', 'affairs', 'adel', 'al', 'jubeir', 'weapons', 'used', 'iranian', 'vowed', 'release', 'full', 'findings', 'investigation', 'iran', 'denies', 'involvement', 'attacks', 'earlier', 'senior', 'iranian', 'military', 'official', 'iran', 'ready', 'destroy', 'aggressor', 'us', 'announced', 'sending', 'troops', 'saudi', 'arabia', 'iranian', 'backed', 'houthi', 'rebels', 'yemen', 'responsible', 'drone', 'missile', 'strikes', '14', 'september', 'affected', 'global', 'oil', 'supply', 'tensions', 'us', 'iran', 'escalated', 'us', 'president', 'donald', 'trump', 'abandoned', 'deal', 'limiting', 'iran', 's', 'nuclear', 'activities', 'last', 'year', 'reinstated', 'sanctions', 'speaking', 'reporters', 'riyadh', 'mr', 'jubeir', 'saudi', 'arabia', 'consultation', 'its', 'allies', 'necessary', 'suitable', 'measures', 'its', 'investigation', 'complete', 'without', 'giving', 'details', 'possible', 'actions', 'repeated', 'strikes', 'targeting', 'abqaiq', 'oil', 'facility', 'khurais', 'oil', 'field', 'north', 'not', 'yemen', 'not', 'give', 'specific', 'location', 'urged', 'international', 'community', 'stand', 'kingdom', 'calls', 'upon', 'international', 'community', 'assume', 'its', 'responsibility', 'condemning', 'stand', 'behind', 'act', 'firm', 'clear', 'position', 'against', 'reckless', 'behaviour', 'threatens', 'global', 'economy', 'saudi', 'defence', 'ministry', 'showed', 'off', 'wednesday', 'remains', 'drones', 'cruise', 'missiles', 'proving', 'iranian', 'involvement', 'us', 'accused', 'iran', 'behind', 'attacks', 'unnamed', 'senior', 'officials', 'told', 'us', 'media', 'evidence', 'suggests', 'strikes', 'originated', 'south', 'iran', 'friday', 'secretary', 'defence', 'mark', 'esper', 'us', 'send', 'yet', 'decided', 'number', 'troops', 'saudi', 'arabia', 'boost', 'country', 's', 'air', 'missile', 'defences', 'president', 'trump', 'announced', 'new', 'sanctions', 'against', 'iran', 'focusing', 'country', 's', 'central', 'bank', 'its', 'sovereign', 'wealth', 'fund', 'signalling', 'wanted', 'avoid', 'military', 'conflict', 'news', 'conference', 'glittering', 'halls', 'saudi', 'arabia', 's', 'foreign', 'ministry', 'fairly', 'predictable', 'its', 'customary', 'condemnation', 'iran', 'islamic', 'republic', 'eloquent', 'softly', 'spoken', 'minister', 'state', 'foreign', 'affairs', 'blame', 'mischief', 'making', 'region', 'including', '260', 'ballistic'])
t.deepEqual(bodyStopped, ['saudi', 'arabia', 'says', 'will', 'respond', 'necessary', 'measures', 'attacks', 'two', 'oil', 'facilities', 'reiterated', 'accusation', 'iran', 'behind', 'minister', 'state', 'foreign', 'affairs', 'adel', 'al', 'jubeir', 'weapons', 'used', 'iranian', 'vowed', 'release', 'full', 'findings', 'investigation', 'iran', 'denies', 'involvement', 'attacks', 'earlier', 'senior', 'iranian', 'military', 'official', 'iran', 'ready', 'destroy', 'aggressor', 'us', 'announced', 'sending', 'troops', 'saudi', 'arabia', 'iranian', 'backed', 'houthi', 'rebels', 'yemen', 'responsible', 'drone', 'missile', 'strikes', '14', 'september', 'affected', 'global', 'oil', 'supply', 'tensions', 'us', 'iran', 'escalated', 'us', 'president', 'donald', 'trump', 'abandoned', 'deal', 'limiting', 'iran', 's', 'nuclear', 'activities', 'last', 'year', 'reinstated', 'sanctions', 'speaking', 'reporters', 'riyadh', 'mr', 'jubeir', 'saudi', 'arabia', 'consultation', 'its', 'allies', 'necessary', 'suitable', 'measures', 'its', 'investigation', 'complete', 'without', 'giving', 'details', 'possible', 'actions', 'repeated', 'strikes', 'targeting', 'abqaiq', 'oil', 'facility', 'khurais', 'oil', 'field', 'north', 'not', 'yemen', 'not', 'give', 'specific', 'location', 'urged', 'international', 'community', 'stand', 'kingdom', 'calls', 'upon', 'international', 'community', 'assume', 'its', 'responsibility', 'condemning', 'stand', 'behind', 'act', 'firm', 'clear', 'position', 'against', 'reckless', 'behaviour', 'threatens', 'global', 'economy', 'saudi', 'defence', 'ministry', 'showed', 'off', 'wednesday', 'remains', 'drones', 'cruise', 'missiles', 'proving', 'iranian', 'involvement', 'us', 'accused', 'iran', 'behind', 'attacks', 'unnamed', 'senior', 'officials', 'told', 'us', 'media', 'evidence', 'suggests', 'strikes', 'originated', 'south', 'iran', 'friday', 'secretary', 'defence', 'mark', 'esper', 'us', 'send', 'yet', 'decided', 'number', 'troops', 'saudi', 'arabia', 'boost', 'country', 's', 'air', 'missile', 'defences', 'president', 'trump', 'announced', 'new', 'sanctions', 'against', 'iran', 'focusing', 'country', 's', 'central', 'bank', 'its', 'sovereign', 'wealth', 'fund', 'signalling', 'wanted', 'avoid', 'military', 'conflict', 'news', 'conference', 'glittering', 'halls', 'saudi', 'arabia', 's', 'foreign', 'ministry', 'fairly', 'predictable', 'its', 'customary', 'condemnation', 'iran', 'islamic', 'republic', 'eloquent', 'softly', 'spoken', 'minister', 'state', 'foreign', 'affairs', 'blame', 'mischief', 'making', 'region', 'including', '260', 'ballistic'])
// Calculate n-grams (2-grams, 3-grams and 4-grams) from title / headline
const headlineNgrams = ngraminator(headlineStopped, [2, 3, 4])
t.looseEqual(headlineNgrams, [['arabia', 'vows'], ['arabia', 'vows', 'respond'], ['arabia', 'vows', 'respond', 'oil'], ['attacks', 'necessary'], ['attacks', 'necessary', 'measures'], ['necessary', 'measures'], ['oil', 'attacks'], ['oil', 'attacks', 'necessary'], ['oil', 'attacks', 'necessary', 'measures'], ['respond', 'oil'], ['respond', 'oil', 'attacks'], ['respond', 'oil', 'attacks', 'necessary'], ['saudi', 'arabia'], ['saudi', 'arabia', 'vows'], ['saudi', 'arabia', 'vows', 'respond'], ['vows', 'respond'], ['vows', 'respond', 'oil'], ['vows', 'respond', 'oil', 'attacks']])
t.deepEqual(headlineNgrams, [['arabia', 'vows'], ['arabia', 'vows', 'respond'], ['arabia', 'vows', 'respond', 'oil'], ['attacks', 'necessary'], ['attacks', 'necessary', 'measures'], ['necessary', 'measures'], ['oil', 'attacks'], ['oil', 'attacks', 'necessary'], ['oil', 'attacks', 'necessary', 'measures'], ['respond', 'oil'], ['respond', 'oil', 'attacks'], ['respond', 'oil', 'attacks', 'necessary'], ['saudi', 'arabia'], ['saudi', 'arabia', 'vows'], ['saudi', 'arabia', 'vows', 'respond'], ['vows', 'respond'], ['vows', 'respond', 'oil'], ['vows', 'respond', 'oil', 'attacks']])
// Calculate n-grams (3-grams) from bodytext / story
const bodyNgrams = ngraminator(bodyStopped, [3])
t.looseEqual(bodyNgrams, [['14', 'september', 'affected'], ['abandoned', 'deal', 'limiting'], ['abqaiq', 'oil', 'facility'], ['accusation', 'iran', 'behind'], ['accused', 'iran', 'behind'], ['act', 'firm', 'clear'], ['actions', 'repeated', 'strikes'], ['activities', 'last', 'year'], ['adel', 'al', 'jubeir'], ['affairs', 'adel', 'al'], ['affairs', 'blame', 'mischief'], ['affected', 'global', 'oil'], ['against', 'iran', 'focusing'], ['against', 'reckless', 'behaviour'], ['aggressor', 'us', 'announced'], ['air', 'missile', 'defences'], ['al', 'jubeir', 'weapons'], ['allies', 'necessary', 'suitable'], ['announced', 'new', 'sanctions'], ['announced', 'sending', 'troops'], ['arabia', 'boost', 'country'], ['arabia', 'consultation', 'its'], ['arabia', 'iranian', 'backed'], ['arabia', 's', 'foreign'], ['arabia', 'says', 'will'], ['assume', 'its', 'responsibility'], ['attacks', 'earlier', 'senior'], ['attacks', 'two', 'oil'], ['attacks', 'unnamed', 'senior'], ['avoid', 'military', 'conflict'], ['backed', 'houthi', 'rebels'], ['bank', 'its', 'sovereign'], ['behaviour', 'threatens', 'global'], ['behind', 'act', 'firm'], ['behind', 'attacks', 'unnamed'], ['behind', 'minister', 'state'], ['blame', 'mischief', 'making'], ['boost', 'country', 's'], ['calls', 'upon', 'international'], ['central', 'bank', 'its'], ['clear', 'position', 'against'], ['community', 'assume', 'its'], ['community', 'stand', 'kingdom'], ['complete', 'without', 'giving'], ['condemnation', 'iran', 'islamic'], ['condemning', 'stand', 'behind'], ['conference', 'glittering', 'halls'], ['conflict', 'news', 'conference'], ['consultation', 'its', 'allies'], ['country', 's', 'air'], ['country', 's', 'central'], ['cruise', 'missiles', 'proving'], ['customary', 'condemnation', 'iran'], ['deal', 'limiting', 'iran'], ['decided', 'number', 'troops'], ['defence', 'mark', 'esper'], ['defence', 'ministry', 'showed'], ['defences', 'president', 'trump'], ['denies', 'involvement', 'attacks'], ['destroy', 'aggressor', 'us'], ['details', 'possible', 'actions'], ['donald', 'trump', 'abandoned'], ['drone', 'missile', 'strikes'], ['drones', 'cruise', 'missiles'], ['earlier', 'senior', 'iranian'], ['economy', 'saudi', 'defence'], ['eloquent', 'softly', 'spoken'], ['escalated', 'us', 'president'], ['esper', 'us', 'send'], ['evidence', 'suggests', 'strikes'], ['facilities', 'reiterated', 'accusation'], ['facility', 'khurais', 'oil'], ['fairly', 'predictable', 'its'], ['field', 'north', 'not'], ['findings', 'investigation', 'iran'], ['firm', 'clear', 'position'], ['focusing', 'country', 's'], ['foreign', 'affairs', 'adel'], ['foreign', 'affairs', 'blame'], ['foreign', 'ministry', 'fairly'], ['friday', 'secretary', 'defence'], ['full', 'findings', 'investigation'], ['fund', 'signalling', 'wanted'], ['give', 'specific', 'location'], ['giving', 'details', 'possible'], ['glittering', 'halls', 'saudi'], ['global', 'economy', 'saudi'], ['global', 'oil', 'supply'], ['halls', 'saudi', 'arabia'], ['houthi', 'rebels', 'yemen'], ['including', '260', 'ballistic'], ['international', 'community', 'assume'], ['international', 'community', 'stand'], ['investigation', 'complete', 'without'], ['investigation', 'iran', 'denies'], ['involvement', 'attacks', 'earlier'], ['involvement', 'us', 'accused'], ['iran', 'behind', 'attacks'], ['iran', 'behind', 'minister'], ['iran', 'denies', 'involvement'], ['iran', 'escalated', 'us'], ['iran', 'focusing', 'country'], ['iran', 'friday', 'secretary'], ['iran', 'islamic', 'republic'], ['iran', 'ready', 'destroy'], ['iran', 's', 'nuclear'], ['iranian', 'backed', 'houthi'], ['iranian', 'involvement', 'us'], ['iranian', 'military', 'official'], ['iranian', 'vowed', 'release'], ['islamic', 'republic', 'eloquent'], ['its', 'allies', 'necessary'], ['its', 'customary', 'condemnation'], ['its', 'investigation', 'complete'], ['its', 'responsibility', 'condemning'], ['its', 'sovereign', 'wealth'], ['jubeir', 'saudi', 'arabia'], ['jubeir', 'weapons', 'used'], ['khurais', 'oil', 'field'], ['kingdom', 'calls', 'upon'], ['last', 'year', 'reinstated'], ['limiting', 'iran', 's'], ['location', 'urged', 'international'], ['making', 'region', 'including'], ['mark', 'esper', 'us'], ['measures', 'attacks', 'two'], ['measures', 'its', 'investigation'], ['media', 'evidence', 'suggests'], ['military', 'conflict', 'news'], ['military', 'official', 'iran'], ['minister', 'state', 'foreign'], ['minister', 'state', 'foreign'], ['ministry', 'fairly', 'predictable'], ['ministry', 'showed', 'off'], ['mischief', 'making', 'region'], ['missile', 'defences', 'president'], ['missile', 'strikes', '14'], ['missiles', 'proving', 'iranian'], ['mr', 'jubeir', 'saudi'], ['necessary', 'measures', 'attacks'], ['necessary', 'suitable', 'measures'], ['new', 'sanctions', 'against'], ['news', 'conference', 'glittering'], ['north', 'not', 'yemen'], ['not', 'give', 'specific'], ['not', 'yemen', 'not'], ['nuclear', 'activities', 'last'], ['number', 'troops', 'saudi'], ['off', 'wednesday', 'remains'], ['official', 'iran', 'ready'], ['officials', 'told', 'us'], ['oil', 'facilities', 'reiterated'], ['oil', 'facility', 'khurais'], ['oil', 'field', 'north'], ['oil', 'supply', 'tensions'], ['originated', 'south', 'iran'], ['position', 'against', 'reckless'], ['possible', 'actions', 'repeated'], ['predictable', 'its', 'customary'], ['president', 'donald', 'trump'], ['president', 'trump', 'announced'], ['proving', 'iranian', 'involvement'], ['ready', 'destroy', 'aggressor'], ['rebels', 'yemen', 'responsible'], ['reckless', 'behaviour', 'threatens'], ['region', 'including', '260'], ['reinstated', 'sanctions', 'speaking'], ['reiterated', 'accusation', 'iran'], ['release', 'full', 'findings'], ['remains', 'drones', 'cruise'], ['repeated', 'strikes', 'targeting'], ['reporters', 'riyadh', 'mr'], ['republic', 'eloquent', 'softly'], ['respond', 'necessary', 'measures'], ['responsibility', 'condemning', 'stand'], ['responsible', 'drone', 'missile'], ['riyadh', 'mr', 'jubeir'], ['s', 'air', 'missile'], ['s', 'central', 'bank'], ['s', 'foreign', 'ministry'], ['s', 'nuclear', 'activities'], ['sanctions', 'against', 'iran'], ['sanctions', 'speaking', 'reporters'], ['saudi', 'arabia', 'boost'], ['saudi', 'arabia', 'consultation'], ['saudi', 'arabia', 'iranian'], ['saudi', 'arabia', 's'], ['saudi', 'arabia', 'says'], ['saudi', 'defence', 'ministry'], ['says', 'will', 'respond'], ['secretary', 'defence', 'mark'], ['send', 'yet', 'decided'], ['sending', 'troops', 'saudi'], ['senior', 'iranian', 'military'], ['senior', 'officials', 'told'], ['september', 'affected', 'global'], ['showed', 'off', 'wednesday'], ['signalling', 'wanted', 'avoid'], ['softly', 'spoken', 'minister'], ['south', 'iran', 'friday'], ['sovereign', 'wealth', 'fund'], ['speaking', 'reporters', 'riyadh'], ['specific', 'location', 'urged'], ['spoken', 'minister', 'state'], ['stand', 'behind', 'act'], ['stand', 'kingdom', 'calls'], ['state', 'foreign', 'affairs'], ['state', 'foreign', 'affairs'], ['strikes', '14', 'september'], ['strikes', 'originated', 'south'], ['strikes', 'targeting', 'abqaiq'], ['suggests', 'strikes', 'originated'], ['suitable', 'measures', 'its'], ['supply', 'tensions', 'us'], ['targeting', 'abqaiq', 'oil'], ['tensions', 'us', 'iran'], ['threatens', 'global', 'economy'], ['told', 'us', 'media'], ['troops', 'saudi', 'arabia'], ['troops', 'saudi', 'arabia'], ['trump', 'abandoned', 'deal'], ['trump', 'announced', 'new'], ['two', 'oil', 'facilities'], ['unnamed', 'senior', 'officials'], ['upon', 'international', 'community'], ['urged', 'international', 'community'], ['us', 'accused', 'iran'], ['us', 'announced', 'sending'], ['us', 'iran', 'escalated'], ['us', 'media', 'evidence'], ['us', 'president', 'donald'], ['us', 'send', 'yet'], ['used', 'iranian', 'vowed'], ['vowed', 'release', 'full'], ['wanted', 'avoid', 'military'], ['wealth', 'fund', 'signalling'], ['weapons', 'used', 'iranian'], ['wednesday', 'remains', 'drones'], ['will', 'respond', 'necessary'], ['without', 'giving', 'details'], ['year', 'reinstated', 'sanctions'], ['yemen', 'not', 'give'], ['yemen', 'responsible', 'drone'], ['yet', 'decided', 'number']])
t.deepEqual(bodyNgrams, [['14', 'september', 'affected'], ['abandoned', 'deal', 'limiting'], ['abqaiq', 'oil', 'facility'], ['accusation', 'iran', 'behind'], ['accused', 'iran', 'behind'], ['act', 'firm', 'clear'], ['actions', 'repeated', 'strikes'], ['activities', 'last', 'year'], ['adel', 'al', 'jubeir'], ['affairs', 'adel', 'al'], ['affairs', 'blame', 'mischief'], ['affected', 'global', 'oil'], ['against', 'iran', 'focusing'], ['against', 'reckless', 'behaviour'], ['aggressor', 'us', 'announced'], ['air', 'missile', 'defences'], ['al', 'jubeir', 'weapons'], ['allies', 'necessary', 'suitable'], ['announced', 'new', 'sanctions'], ['announced', 'sending', 'troops'], ['arabia', 'boost', 'country'], ['arabia', 'consultation', 'its'], ['arabia', 'iranian', 'backed'], ['arabia', 's', 'foreign'], ['arabia', 'says', 'will'], ['assume', 'its', 'responsibility'], ['attacks', 'earlier', 'senior'], ['attacks', 'two', 'oil'], ['attacks', 'unnamed', 'senior'], ['avoid', 'military', 'conflict'], ['backed', 'houthi', 'rebels'], ['bank', 'its', 'sovereign'], ['behaviour', 'threatens', 'global'], ['behind', 'act', 'firm'], ['behind', 'attacks', 'unnamed'], ['behind', 'minister', 'state'], ['blame', 'mischief', 'making'], ['boost', 'country', 's'], ['calls', 'upon', 'international'], ['central', 'bank', 'its'], ['clear', 'position', 'against'], ['community', 'assume', 'its'], ['community', 'stand', 'kingdom'], ['complete', 'without', 'giving'], ['condemnation', 'iran', 'islamic'], ['condemning', 'stand', 'behind'], ['conference', 'glittering', 'halls'], ['conflict', 'news', 'conference'], ['consultation', 'its', 'allies'], ['country', 's', 'air'], ['country', 's', 'central'], ['cruise', 'missiles', 'proving'], ['customary', 'condemnation', 'iran'], ['deal', 'limiting', 'iran'], ['decided', 'number', 'troops'], ['defence', 'mark', 'esper'], ['defence', 'ministry', 'showed'], ['defences', 'president', 'trump'], ['denies', 'involvement', 'attacks'], ['destroy', 'aggressor', 'us'], ['details', 'possible', 'actions'], ['donald', 'trump', 'abandoned'], ['drone', 'missile', 'strikes'], ['drones', 'cruise', 'missiles'], ['earlier', 'senior', 'iranian'], ['economy', 'saudi', 'defence'], ['eloquent', 'softly', 'spoken'], ['escalated', 'us', 'president'], ['esper', 'us', 'send'], ['evidence', 'suggests', 'strikes'], ['facilities', 'reiterated', 'accusation'], ['facility', 'khurais', 'oil'], ['fairly', 'predictable', 'its'], ['field', 'north', 'not'], ['findings', 'investigation', 'iran'], ['firm', 'clear', 'position'], ['focusing', 'country', 's'], ['foreign', 'affairs', 'adel'], ['foreign', 'affairs', 'blame'], ['foreign', 'ministry', 'fairly'], ['friday', 'secretary', 'defence'], ['full', 'findings', 'investigation'], ['fund', 'signalling', 'wanted'], ['give', 'specific', 'location'], ['giving', 'details', 'possible'], ['glittering', 'halls', 'saudi'], ['global', 'economy', 'saudi'], ['global', 'oil', 'supply'], ['halls', 'saudi', 'arabia'], ['houthi', 'rebels', 'yemen'], ['including', '260', 'ballistic'], ['international', 'community', 'assume'], ['international', 'community', 'stand'], ['investigation', 'complete', 'without'], ['investigation', 'iran', 'denies'], ['involvement', 'attacks', 'earlier'], ['involvement', 'us', 'accused'], ['iran', 'behind', 'attacks'], ['iran', 'behind', 'minister'], ['iran', 'denies', 'involvement'], ['iran', 'escalated', 'us'], ['iran', 'focusing', 'country'], ['iran', 'friday', 'secretary'], ['iran', 'islamic', 'republic'], ['iran', 'ready', 'destroy'], ['iran', 's', 'nuclear'], ['iranian', 'backed', 'houthi'], ['iranian', 'involvement', 'us'], ['iranian', 'military', 'official'], ['iranian', 'vowed', 'release'], ['islamic', 'republic', 'eloquent'], ['its', 'allies', 'necessary'], ['its', 'customary', 'condemnation'], ['its', 'investigation', 'complete'], ['its', 'responsibility', 'condemning'], ['its', 'sovereign', 'wealth'], ['jubeir', 'saudi', 'arabia'], ['jubeir', 'weapons', 'used'], ['khurais', 'oil', 'field'], ['kingdom', 'calls', 'upon'], ['last', 'year', 'reinstated'], ['limiting', 'iran', 's'], ['location', 'urged', 'international'], ['making', 'region', 'including'], ['mark', 'esper', 'us'], ['measures', 'attacks', 'two'], ['measures', 'its', 'investigation'], ['media', 'evidence', 'suggests'], ['military', 'conflict', 'news'], ['military', 'official', 'iran'], ['minister', 'state', 'foreign'], ['minister', 'state', 'foreign'], ['ministry', 'fairly', 'predictable'], ['ministry', 'showed', 'off'], ['mischief', 'making', 'region'], ['missile', 'defences', 'president'], ['missile', 'strikes', '14'], ['missiles', 'proving', 'iranian'], ['mr', 'jubeir', 'saudi'], ['necessary', 'measures', 'attacks'], ['necessary', 'suitable', 'measures'], ['new', 'sanctions', 'against'], ['news', 'conference', 'glittering'], ['north', 'not', 'yemen'], ['not', 'give', 'specific'], ['not', 'yemen', 'not'], ['nuclear', 'activities', 'last'], ['number', 'troops', 'saudi'], ['off', 'wednesday', 'remains'], ['official', 'iran', 'ready'], ['officials', 'told', 'us'], ['oil', 'facilities', 'reiterated'], ['oil', 'facility', 'khurais'], ['oil', 'field', 'north'], ['oil', 'supply', 'tensions'], ['originated', 'south', 'iran'], ['position', 'against', 'reckless'], ['possible', 'actions', 'repeated'], ['predictable', 'its', 'customary'], ['president', 'donald', 'trump'], ['president', 'trump', 'announced'], ['proving', 'iranian', 'involvement'], ['ready', 'destroy', 'aggressor'], ['rebels', 'yemen', 'responsible'], ['reckless', 'behaviour', 'threatens'], ['region', 'including', '260'], ['reinstated', 'sanctions', 'speaking'], ['reiterated', 'accusation', 'iran'], ['release', 'full', 'findings'], ['remains', 'drones', 'cruise'], ['repeated', 'strikes', 'targeting'], ['reporters', 'riyadh', 'mr'], ['republic', 'eloquent', 'softly'], ['respond', 'necessary', 'measures'], ['responsibility', 'condemning', 'stand'], ['responsible', 'drone', 'missile'], ['riyadh', 'mr', 'jubeir'], ['s', 'air', 'missile'], ['s', 'central', 'bank'], ['s', 'foreign', 'ministry'], ['s', 'nuclear', 'activities'], ['sanctions', 'against', 'iran'], ['sanctions', 'speaking', 'reporters'], ['saudi', 'arabia', 'boost'], ['saudi', 'arabia', 'consultation'], ['saudi', 'arabia', 'iranian'], ['saudi', 'arabia', 's'], ['saudi', 'arabia', 'says'], ['saudi', 'defence', 'ministry'], ['says', 'will', 'respond'], ['secretary', 'defence', 'mark'], ['send', 'yet', 'decided'], ['sending', 'troops', 'saudi'], ['senior', 'iranian', 'military'], ['senior', 'officials', 'told'], ['september', 'affected', 'global'], ['showed', 'off', 'wednesday'], ['signalling', 'wanted', 'avoid'], ['softly', 'spoken', 'minister'], ['south', 'iran', 'friday'], ['sovereign', 'wealth', 'fund'], ['speaking', 'reporters', 'riyadh'], ['specific', 'location', 'urged'], ['spoken', 'minister', 'state'], ['stand', 'behind', 'act'], ['stand', 'kingdom', 'calls'], ['state', 'foreign', 'affairs'], ['state', 'foreign', 'affairs'], ['strikes', '14', 'september'], ['strikes', 'originated', 'south'], ['strikes', 'targeting', 'abqaiq'], ['suggests', 'strikes', 'originated'], ['suitable', 'measures', 'its'], ['supply', 'tensions', 'us'], ['targeting', 'abqaiq', 'oil'], ['tensions', 'us', 'iran'], ['threatens', 'global', 'economy'], ['told', 'us', 'media'], ['troops', 'saudi', 'arabia'], ['troops', 'saudi', 'arabia'], ['trump', 'abandoned', 'deal'], ['trump', 'announced', 'new'], ['two', 'oil', 'facilities'], ['unnamed', 'senior', 'officials'], ['upon', 'international', 'community'], ['urged', 'international', 'community'], ['us', 'accused', 'iran'], ['us', 'announced', 'sending'], ['us', 'iran', 'escalated'], ['us', 'media', 'evidence'], ['us', 'president', 'donald'], ['us', 'send', 'yet'], ['used', 'iranian', 'vowed'], ['vowed', 'release', 'full'], ['wanted', 'avoid', 'military'], ['wealth', 'fund', 'signalling'], ['weapons', 'used', 'iranian'], ['wednesday', 'remains', 'drones'], ['will', 'respond', 'necessary'], ['without', 'giving', 'details'], ['year', 'reinstated', 'sanctions'], ['yemen', 'not', 'give'], ['yemen', 'responsible', 'drone'], ['yet', 'decided', 'number']])
// Calculate keywords in title based on bodytext context
const keywords = ehp.findKeywords(headlineStopped, bodyStopped, 5)
t.looseEqual(keywords, ['saudi', 'arabia', 'oil', 'attacks', 'necessary'])
t.deepEqual(keywords, ['saudi', 'arabia', 'oil', 'attacks', 'necessary'])

@@ -48,3 +48,3 @@ // Highlight a result item where query words matches

const highlighted = highlight(query, headlineArray)
t.looseEqual(highlighted, ['<span class="highlighted">saudi</span>', '<span class="highlighted">arabia</span>', 'vows', 'to', 'respond', 'to', 'oil', '<span class="highlighted">attacks</span>', 'with', 'necessary', 'measures'])
t.deepEqual(highlighted, '<span class="hitHighlight">saudi arabia</span> vows to respond to oil <span class="hitHighlight">attacks</span> with necessary measures ')
})
demo/index.html

Sorry, the diff of this file is not supported yet

dist/daq-proc.latest.js

Sorry, the diff of this file is too big to display

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc