atomic-algolia
Advanced tools
Comparing version 0.3.12 to 0.3.13
#!/usr/bin/env node | ||
var dotenv = require("dotenv").config() | ||
var update = require("./update") | ||
"use strict"; | ||
require("source-map-support/register"); | ||
var dotenv = require("dotenv").config(); | ||
var update = require("./update"); | ||
if (require.main === module) { | ||
var index = process.env.ALGOLIA_INDEX_NAME | ||
var indexFile = process.env.ALGOLIA_INDEX_FILE | ||
var index = process.env.ALGOLIA_INDEX_NAME; | ||
var indexFile = process.env.ALGOLIA_INDEX_FILE; | ||
if (!index) | ||
throw new Error("Please provide `process.env.ALGOLIA_INDEX_NAME`") | ||
if (!indexFile) | ||
throw new Error("Please provide `process.env.ALGOLIA_INDEX_FILE`") | ||
if (!index) throw new Error("Please provide `process.env.ALGOLIA_INDEX_NAME`"); | ||
update(index, indexFile, {verbose: true}, function(err, result) { | ||
if (err) throw err | ||
if (!indexFile) throw new Error("Please provide `process.env.ALGOLIA_INDEX_FILE`"); | ||
console.log(result) | ||
}) | ||
update(index, indexFile, { verbose: true }, function (err, result) { | ||
if (err) throw err; | ||
console.log(result); | ||
}); | ||
} else { | ||
module.exports = update | ||
} | ||
module.exports = update; | ||
} |
@@ -1,88 +0,85 @@ | ||
var actionAdd = require("./utils/actionAdd") | ||
var actionUpdate = require("./utils/actionUpdate") | ||
var actionDelete = require("./utils/actionDelete") | ||
var algoliaSearch = require("algoliasearch") | ||
var chalk = require("chalk") | ||
var dotenv = require("dotenv").config() | ||
var fs = require("fs") | ||
var getLocalIndex = require("./utils/getLocalIndex") | ||
var getRemoteIndex = require("./utils/getRemoteIndex") | ||
var calculateOperations = require("./utils/calculateOperations") | ||
var title = "[" + chalk.blue("Algolia") + "]" | ||
"use strict"; | ||
require("source-map-support/register"); | ||
var actionAdd = require("./utils/actionAdd"); | ||
var actionUpdate = require("./utils/actionUpdate"); | ||
var actionDelete = require("./utils/actionDelete"); | ||
var algoliaSearch = require("algoliasearch"); | ||
var chalk = require("chalk"); | ||
var dotenv = require("dotenv").config(); | ||
var fs = require("fs"); | ||
var getLocalIndex = require("./utils/getLocalIndex"); | ||
var getRemoteIndex = require("./utils/getRemoteIndex"); | ||
var calculateOperations = require("./utils/calculateOperations"); | ||
var title = "[" + chalk.blue("Algolia") + "]"; | ||
module.exports = function update(indexName, indexData, options, cb) { | ||
try { | ||
if (!indexName) | ||
throw new Error("Please provide `indexName`") | ||
if (!indexName) throw new Error("Please provide `indexName`"); | ||
if (!indexData) | ||
throw new Error("Please provide `indexData`. A valid Javacript object or path to a JSON file.") | ||
if (!indexData) throw new Error("Please provide `indexData`. A valid Javacript object or path to a JSON file."); | ||
if (typeof options === "function" && typeof cb !== "function") { | ||
cb = options | ||
cb = options; | ||
} | ||
var client = algoliaSearch( | ||
process.env.ALGOLIA_APP_ID, | ||
process.env.ALGOLIA_ADMIN_KEY | ||
) | ||
var index = client.initIndex(indexName) | ||
var newIndex = getLocalIndex(indexData) | ||
return getRemoteIndex(index) | ||
.then(function(oldIndex) { | ||
// Figure out which records to add or delete | ||
var operations = calculateOperations(newIndex, oldIndex) | ||
var client = algoliaSearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_ADMIN_KEY); | ||
if (options.verbose === true) { | ||
console.log(title, `Adding ${operations.add.length} hits to ${indexName}`) | ||
console.log(title, `Updating ${operations.update.length} hits to ${indexName}`) | ||
console.log(title, `Removing ${operations.delete.length} hits from ${indexName}`) | ||
console.log(title, `${operations.ignore.length} hits unchanged in ${indexName}`) | ||
} | ||
var index = client.initIndex(indexName); | ||
var newIndex = getLocalIndex(indexData); | ||
// Fetch full records from operation ids | ||
var toAddRecords = newIndex.filter(function(hit) { | ||
return operations.add.indexOf(hit.objectID) !== -1 | ||
}) | ||
return getRemoteIndex(index).then(function (oldIndex) { | ||
// Figure out which records to add or delete | ||
var operations = calculateOperations(newIndex, oldIndex); | ||
var toUpdateRecords = newIndex.filter(function(hit) { | ||
return operations.update.indexOf(hit.objectID) !== -1 | ||
}) | ||
if (options.verbose === true) { | ||
console.log(title, "Adding " + operations.add.length + " hits to " + indexName); | ||
console.log(title, "Updating " + operations.update.length + " hits to " + indexName); | ||
console.log(title, "Removing " + operations.delete.length + " hits from " + indexName); | ||
console.log(title, operations.ignore.length + " hits unchanged in " + indexName); | ||
} | ||
// Create batch update actions | ||
var toAdd = toAddRecords.map(function(record) { | ||
return actionAdd(record, indexName) | ||
}) | ||
// Fetch full records from operation ids | ||
var toAddRecords = newIndex.filter(function (hit) { | ||
return operations.add.indexOf(hit.objectID) !== -1; | ||
}); | ||
var toUpdate = toUpdateRecords.map(function(record) { | ||
return actionUpdate(record, indexName) | ||
}) | ||
var toUpdateRecords = newIndex.filter(function (hit) { | ||
return operations.update.indexOf(hit.objectID) !== -1; | ||
}); | ||
var toDelete = operations.delete.map(function(id) { | ||
return actionDelete(id, indexName) | ||
}) | ||
var batchActions = [].concat(toAdd, toUpdate, toDelete) | ||
// Create batch update actions | ||
var toAdd = toAddRecords.map(function (record) { | ||
return actionAdd(record, indexName); | ||
}); | ||
// Perform the batch API call | ||
if (batchActions.length > 0) { | ||
// Notify client this is coming from this script | ||
client.setExtraHeader("X-FORWARDED-BY", "ATOMIC-ALGOLIA") | ||
client.batch(batchActions, function(err, res) { | ||
if (err) throw err | ||
var toUpdate = toUpdateRecords.map(function (record) { | ||
return actionUpdate(record, indexName); | ||
}); | ||
cb(null, res) | ||
}) | ||
} else { | ||
cb(null, {}) | ||
} | ||
}) | ||
.catch(function(err) { | ||
cb(err) | ||
}) | ||
var toDelete = operations.delete.map(function (id) { | ||
return actionDelete(id, indexName); | ||
}); | ||
var batchActions = [].concat(toAdd, toUpdate, toDelete); | ||
// Perform the batch API call | ||
if (batchActions.length > 0) { | ||
// Notify client this is coming from this script | ||
client.setExtraHeader("X-FORWARDED-BY", "ATOMIC-ALGOLIA"); | ||
client.batch(batchActions, function (err, res) { | ||
if (err) throw err; | ||
cb(null, res); | ||
}); | ||
} else { | ||
cb(null, {}); | ||
} | ||
}).catch(function (err) { | ||
cb(err); | ||
}); | ||
} catch (err) { | ||
cb(err) | ||
cb(err); | ||
} | ||
} | ||
}; |
@@ -0,1 +1,5 @@ | ||
"use strict"; | ||
require("source-map-support/register"); | ||
module.exports = function createAction(hit, indexName) { | ||
@@ -6,3 +10,3 @@ return { | ||
body: hit | ||
} | ||
} | ||
}; | ||
}; |
@@ -0,1 +1,5 @@ | ||
"use strict"; | ||
require("source-map-support/register"); | ||
module.exports = function deleteAction(id, indexName) { | ||
@@ -8,3 +12,3 @@ return { | ||
} | ||
} | ||
} | ||
}; | ||
}; |
@@ -0,1 +1,5 @@ | ||
"use strict"; | ||
require("source-map-support/register"); | ||
module.exports = function updateAction(hit, indexName) { | ||
@@ -6,3 +10,3 @@ return { | ||
body: hit | ||
} | ||
} | ||
}; | ||
}; |
@@ -1,91 +0,104 @@ | ||
var idsFromIndex = require("./idsFromIndex") | ||
var md5 = require("md5") | ||
"use strict"; | ||
var _keys = require("babel-runtime/core-js/object/keys"); | ||
var _keys2 = _interopRequireDefault(_keys); | ||
var _stringify = require("babel-runtime/core-js/json/stringify"); | ||
var _stringify2 = _interopRequireDefault(_stringify); | ||
require("source-map-support/register"); | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } | ||
var idsFromIndex = require("./idsFromIndex"); | ||
var md5 = require("md5"); | ||
module.exports = function calculateOperations(newIndex, oldIndex) { | ||
var newIndexIds = [] | ||
var oldIndexIds = [] | ||
var newIndexIds = []; | ||
var oldIndexIds = []; | ||
if (Array.isArray(newIndex)) { | ||
newIndexIds = idsFromIndex(newIndex) | ||
newIndexIds = idsFromIndex(newIndex); | ||
} | ||
if (Array.isArray(oldIndex)) { | ||
oldIndexIds = idsFromIndex(oldIndex) | ||
oldIndexIds = idsFromIndex(oldIndex); | ||
} | ||
var existingHits = []; | ||
var operations = { ignore: [], update: [], add: newIndexIds, delete: oldIndexIds }; | ||
var existingHits = [] | ||
var operations = {ignore: [], update: [], add: newIndexIds, delete: oldIndexIds} | ||
if (newIndexIds.length > 0 && oldIndexIds.length > 0) { | ||
existingHits = findExistingHits(newIndexIds, oldIndexIds) | ||
operations.add = findNewHits(newIndexIds, oldIndexIds) | ||
operations.delete = findExpiredHits(newIndexIds, oldIndexIds) | ||
existingHits = findExistingHits(newIndexIds, oldIndexIds); | ||
operations.add = findNewHits(newIndexIds, oldIndexIds); | ||
operations.delete = findExpiredHits(newIndexIds, oldIndexIds); | ||
} | ||
if (existingHits.length > 0) { | ||
operations.ignore = findUnchangedHits(existingHits, newIndex, oldIndex) | ||
operations.update = findChangedHits(existingHits, newIndex, oldIndex) | ||
operations.ignore = findUnchangedHits(existingHits, newIndex, oldIndex); | ||
operations.update = findChangedHits(existingHits, newIndex, oldIndex); | ||
} | ||
return operations | ||
} | ||
return operations; | ||
}; | ||
function findNewHits(newIndexIds, oldIndexIds) { | ||
return newIndexIds.filter(function(id) { | ||
return oldIndexIds.indexOf(id) === -1 | ||
}) | ||
return newIndexIds.filter(function (id) { | ||
return oldIndexIds.indexOf(id) === -1; | ||
}); | ||
} | ||
function findExpiredHits(newIndexIds, oldIndexIds) { | ||
return oldIndexIds.filter(function(id) { | ||
return newIndexIds.indexOf(id) === -1 | ||
}) | ||
return oldIndexIds.filter(function (id) { | ||
return newIndexIds.indexOf(id) === -1; | ||
}); | ||
} | ||
function findExistingHits(newIndexIds, oldIndexIds) { | ||
return newIndexIds.filter(function(id) { | ||
return oldIndexIds.indexOf(id) !== -1 | ||
}) | ||
return newIndexIds.filter(function (id) { | ||
return oldIndexIds.indexOf(id) !== -1; | ||
}); | ||
} | ||
function findUnchangedHits(existingHits, newIndex, oldIndex) { | ||
return existingHits.filter(function(id) { | ||
var shouldUpdate = compareHitFromIndexes(id, newIndex, oldIndex) | ||
return existingHits.filter(function (id) { | ||
var shouldUpdate = compareHitFromIndexes(id, newIndex, oldIndex); | ||
if (shouldUpdate !== true) { | ||
return id | ||
return id; | ||
} | ||
}) | ||
}); | ||
} | ||
function findChangedHits(existingHits, newIndex, oldIndex) { | ||
return existingHits.filter(function(id) { | ||
var shouldUpdate = compareHitFromIndexes(id, newIndex, oldIndex) | ||
return existingHits.filter(function (id) { | ||
var shouldUpdate = compareHitFromIndexes(id, newIndex, oldIndex); | ||
if (shouldUpdate === true) { | ||
return id | ||
return id; | ||
} | ||
}) | ||
}); | ||
} | ||
function compareHitFromIndexes(id, newIndex, oldIndex) { | ||
var newHit = newIndex.filter(function(hit) { | ||
return hit.objectID === id | ||
}) | ||
var newHit = newIndex.filter(function (hit) { | ||
return hit.objectID === id; | ||
}); | ||
var oldHit = oldIndex.filter(function(hit) { | ||
return String(hit.objectID) === String(id) | ||
}) | ||
var oldHit = oldIndex.filter(function (hit) { | ||
return String(hit.objectID) === String(id); | ||
}); | ||
if (newHit.length > 0 && oldHit.length > 0 ) { | ||
var newHitSorted = JSON.stringify(newHit[0], Object.keys(newHit[0]).sort()) | ||
var oldHitSorted = JSON.stringify(oldHit[0], Object.keys(oldHit[0]).sort()) | ||
var newHash = md5(newHitSorted) | ||
var oldHash = md5(oldHitSorted) | ||
if (newHit.length > 0 && oldHit.length > 0) { | ||
var newHitSorted = (0, _stringify2.default)(newHit[0], (0, _keys2.default)(newHit[0]).sort()); | ||
var oldHitSorted = (0, _stringify2.default)(oldHit[0], (0, _keys2.default)(oldHit[0]).sort()); | ||
var newHash = md5(newHitSorted); | ||
var oldHash = md5(oldHitSorted); | ||
return newHash !== oldHash | ||
return newHash !== oldHash; | ||
} | ||
return null | ||
return null; | ||
} |
@@ -1,21 +0,25 @@ | ||
var fs = require("fs") | ||
var path = require("path") | ||
"use strict"; | ||
require("source-map-support/register"); | ||
var fs = require("fs"); | ||
var path = require("path"); | ||
module.exports = function getLocalIndex(indexData) { | ||
if (typeof indexData === "string") { | ||
return getIndexFromFile(indexData) | ||
return getIndexFromFile(indexData); | ||
} else { | ||
return indexData | ||
return indexData; | ||
} | ||
} | ||
}; | ||
function getIndexFromFile(filePath) { | ||
var indexPath = path.resolve(filePath) | ||
var fileContents = fs.readFileSync(indexPath, "utf-8") | ||
var indexPath = path.resolve(filePath); | ||
var fileContents = fs.readFileSync(indexPath, "utf-8"); | ||
if (fileContents !== null && fileContents !== undefined) { | ||
return JSON.parse(fileContents) | ||
return JSON.parse(fileContents); | ||
} else { | ||
return [] | ||
return []; | ||
} | ||
} |
@@ -1,33 +0,78 @@ | ||
module.exports = async function getRemoteIndex(index) { | ||
var oldIndex | ||
try { | ||
oldIndex = await query(index) | ||
} catch (err) { | ||
// do something... | ||
"use strict"; | ||
var _promise = require("babel-runtime/core-js/promise"); | ||
var _promise2 = _interopRequireDefault(_promise); | ||
var _regenerator = require("babel-runtime/regenerator"); | ||
var _regenerator2 = _interopRequireDefault(_regenerator); | ||
var _asyncToGenerator2 = require("babel-runtime/helpers/asyncToGenerator"); | ||
var _asyncToGenerator3 = _interopRequireDefault(_asyncToGenerator2); | ||
require("source-map-support/register"); | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } | ||
module.exports = function () { | ||
var _ref = (0, _asyncToGenerator3.default)( /*#__PURE__*/_regenerator2.default.mark(function _callee(index) { | ||
var oldIndex; | ||
return _regenerator2.default.wrap(function _callee$(_context) { | ||
while (1) { | ||
switch (_context.prev = _context.next) { | ||
case 0: | ||
_context.prev = 0; | ||
_context.next = 3; | ||
return query(index); | ||
case 3: | ||
oldIndex = _context.sent; | ||
_context.next = 8; | ||
break; | ||
case 6: | ||
_context.prev = 6; | ||
_context.t0 = _context["catch"](0); | ||
case 8: | ||
return _context.abrupt("return", oldIndex); | ||
case 9: | ||
case "end": | ||
return _context.stop(); | ||
} | ||
} | ||
}, _callee, this, [[0, 6]]); | ||
})); | ||
function getRemoteIndex(_x) { | ||
return _ref.apply(this, arguments); | ||
} | ||
return oldIndex | ||
} | ||
return getRemoteIndex; | ||
}(); | ||
function query(index) { | ||
return new Promise(function(resolve) { | ||
return new _promise2.default(function (resolve) { | ||
index.browse("", {}, function browseDone(err, content, hits) { | ||
if (err) throw err | ||
if (err) throw err; | ||
if (!Array.isArray(hits)) { | ||
hits = [] | ||
hits = []; | ||
} | ||
content.hits.forEach(function(hit) { | ||
hit.objectID = String(hit.objectID) | ||
hits.push(hit) | ||
}) | ||
content.hits.forEach(function (hit) { | ||
hit.objectID = String(hit.objectID); | ||
hits.push(hit); | ||
}); | ||
if (content.cursor) { | ||
resolve(index.browseFrom(content.cursor, browseDone, hits)) | ||
resolve(index.browseFrom(content.cursor, browseDone, hits)); | ||
} else { | ||
resolve(hits) | ||
resolve(hits); | ||
} | ||
}) | ||
}) | ||
}); | ||
}); | ||
} |
@@ -0,12 +1,16 @@ | ||
"use strict"; | ||
require("source-map-support/register"); | ||
module.exports = function idsFromIndex(index) { | ||
return index.reduce(function(hits, hit) { | ||
if (hit !== null && hit !== undefined) { | ||
return hits.concat(hit) | ||
} | ||
}, []).reduce(function(hits, hit) { | ||
return index.reduce(function (hits, hit) { | ||
if (hit !== null && hit !== undefined) { | ||
return hits.concat(hit); | ||
} | ||
}, []).reduce(function (hits, hit) { | ||
if (hit.objectID !== null && hit.objectID !== undefined) { | ||
var id = String(hit.objectID) | ||
return hits.concat(id) | ||
var id = String(hit.objectID); | ||
return hits.concat(id); | ||
} | ||
}, []).sort() | ||
} | ||
}, []).sort(); | ||
}; |
{ | ||
"name": "atomic-algolia", | ||
"version": "0.3.12", | ||
"description": "An NPM package for running atomic updates to an Algolia index", | ||
"version": "0.3.13", | ||
"description": | ||
"An NPM package for running atomic updates to an Algolia index", | ||
"main": "lib/index.js", | ||
"bin": "lib/index.js", | ||
"scripts": { | ||
"test": "echo \"Error: no test specified\" && exit 1" | ||
"prepublish": "npm run babel", | ||
"babel": "babel src -d lib", | ||
"babel:watch": "npm run babel -- --watch", | ||
"clean": "rimraf lib", | ||
"test": "echo \"Error: no test specified\" && exit 1", | ||
"contributors:add": "all-contributors add", | ||
"contributors:generate": "all-contributors generate", | ||
"contributors:check": "all-contributors check" | ||
}, | ||
"keywords": [ | ||
"algolia", | ||
"node" | ||
], | ||
"keywords": ["algolia", "node"], | ||
"engines": { | ||
@@ -21,3 +26,3 @@ "node": ">=7.6.0" | ||
"email": "hello@chrisdmacrae.com", | ||
"url": "chrisdmacrae.com" | ||
"url": "https://chrisdmacrae.com" | ||
}, | ||
@@ -30,3 +35,21 @@ "license": "ISC", | ||
"md5": "^2.2.1" | ||
}, | ||
"devDependencies": { | ||
"all-contributors-cli": "^4.11.0", | ||
"babel-core": "^6.26.0", | ||
"babel-loader": "^7.1.4", | ||
"babel-plugin-source-map-support": "^2.0.1", | ||
"babel-plugin-transform-object-rest-spread": "^6.26.0", | ||
"babel-plugin-transform-runtime": "^6.23.0", | ||
"babel-polyfill": "^6.26.0", | ||
"babel-preset-env": "^1.6.1", | ||
"babel-runtime": "^6.26.0", | ||
"eslint": "^4.19.0", | ||
"eslint-config-prettier": "^2.9.0", | ||
"prettier": "^1.11.1", | ||
"source-map-support": "^0.5.4", | ||
"stylelint": "^9.1.3", | ||
"stylelint-config-standard": "^18.2.0", | ||
"yarn": "^1.5.1" | ||
} | ||
} |
# Algolia Atomic | ||
[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors) | ||
An NPM package for running atomic updates to Algolia indices | ||
@@ -134,1 +135,13 @@ | ||
`ALGOLIA_INDEX_FILE`: the relative path to your index file from the root of your project. | ||
## Contributors | ||
Thanks goes to these wonderful people ([emoji key](https://github.com/kentcdodds/all-contributors#emoji-key)): | ||
<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section --> | ||
<!-- prettier-ignore --> | ||
| [<img src="https://avatars2.githubusercontent.com/u/6855186?v=4" width="100px;"/><br /><sub><b>chrisdmacrae</b></sub>](https://github.com/chrisdmacrae)<br />[💬](#question-chrisdmacrae "Answering Questions") [💻](https://github.com/algolia/atomic-algolia/commits?author=chrisdmacrae "Code") [🎨](#design-chrisdmacrae "Design") [📖](https://github.com/algolia/atomic-algolia/commits?author=chrisdmacrae "Documentation") | [<img src="https://avatars2.githubusercontent.com/u/283419?v=4" width="100px;"/><br /><sub><b>Tim Carry</b></sub>](http://www.pixelastic.com/)<br />[💬](#question-pixelastic "Answering Questions") [💻](https://github.com/algolia/atomic-algolia/commits?author=pixelastic "Code") [📖](https://github.com/algolia/atomic-algolia/commits?author=pixelastic "Documentation") | | ||
| :---: | :---: | | ||
<!-- ALL-CONTRIBUTORS-LIST:END --> | ||
This project follows the [all-contributors](https://github.com/kentcdodds/all-contributors) specification. Contributions of any kind welcome! |
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
146364
26
510
146
16
13
1