@cumulus/cmr-client
Advanced tools
Comparing version 1.24.0 to 2.0.0
480
CMR.js
'use strict'; | ||
const get = require('lodash/get'); | ||
@@ -8,3 +7,2 @@ const got = require('got'); | ||
const secretsManagerUtils = require('@cumulus/aws-client/SecretsManager'); | ||
const searchConcept = require('./searchConcept'); | ||
@@ -16,15 +14,9 @@ const ingestConcept = require('./ingestConcept'); | ||
const { ummVersion, validateUMMG } = require('./UmmUtils'); | ||
const log = new Logger({ sender: 'cmr-client' }); | ||
const logDetails = { | ||
file: 'cmr-client/CMR.js' | ||
file: 'cmr-client/CMR.js' | ||
}; | ||
const IP_TIMEOUT_MS = 1 * 1000; | ||
const userIpAddress = () => | ||
publicIp.v4({ timeout: IP_TIMEOUT_MS }) | ||
const userIpAddress = () => publicIp.v4({ timeout: IP_TIMEOUT_MS }) | ||
.catch((_) => '127.0.0.1'); | ||
/** | ||
@@ -42,36 +34,33 @@ * Returns a valid a CMR token | ||
async function updateToken(cmrProvider, clientId, username, password) { | ||
// if (!cmrProvider) throw new Error('cmrProvider is required.'); | ||
// if (!clientId) throw new Error('clientId is required.'); | ||
// if (!username) throw new Error('username is required.'); | ||
// if (!password) throw new Error('password is required.'); | ||
// Update the saved ECHO token | ||
// for info on how to add collections to CMR: https://cmr.earthdata.nasa.gov/ingest/site/ingest_api_docs.html#validate-collection | ||
let response; | ||
try { | ||
response = await got.post(getUrl('token'), { | ||
json: true, | ||
body: { | ||
token: { | ||
username: username, | ||
password: password, | ||
client_id: clientId, | ||
user_ip_address: await userIpAddress(), | ||
provider: cmrProvider | ||
// if (!cmrProvider) throw new Error('cmrProvider is required.'); | ||
// if (!clientId) throw new Error('clientId is required.'); | ||
// if (!username) throw new Error('username is required.'); | ||
// if (!password) throw new Error('password is required.'); | ||
// Update the saved ECHO token | ||
// for info on how to add collections to CMR: https://cmr.earthdata.nasa.gov/ingest/site/ingest_api_docs.html#validate-collection | ||
let response; | ||
try { | ||
response = await got.post(getUrl('token'), { | ||
json: true, | ||
body: { | ||
token: { | ||
username: username, | ||
password: password, | ||
client_id: clientId, | ||
user_ip_address: await userIpAddress(), | ||
provider: cmrProvider | ||
} | ||
} | ||
}); | ||
} | ||
catch (error) { | ||
if (get(error, 'response.body.errors')) { | ||
throw new Error(`CMR Error: ${error.response.body.errors[0]}`); | ||
} | ||
} | ||
}); | ||
} catch (err) { | ||
if (get(err, 'response.body.errors')) { | ||
throw new Error(`CMR Error: ${err.response.body.errors[0]}`); | ||
throw error; | ||
} | ||
throw err; | ||
} | ||
if (!response.body.token) throw new Error('Authentication with CMR failed'); | ||
return response.body.token.id; | ||
if (!response.body.token) | ||
throw new Error('Authentication with CMR failed'); | ||
return response.body.token.id; | ||
} | ||
/** | ||
@@ -101,226 +90,195 @@ * A class to simplify requests to the CMR | ||
class CMR { | ||
/** | ||
* The constructor for the CMR class | ||
* | ||
* @param {Object} params | ||
* @param {string} params.provider - the CMR provider id | ||
* @param {string} params.clientId - the CMR clientId | ||
* @param {string} params.username - CMR username, not used if token is provided | ||
* @param {string} params.passwordSecretName - CMR password secret, not used if token is provided | ||
* @param {string} params.password - CMR password, not used if token or | ||
* passwordSecretName is provided | ||
* @param {string} params.token - CMR or Launchpad token, | ||
* if not provided, CMR username and password are used to get a cmr token | ||
*/ | ||
constructor(params = {}) { | ||
this.clientId = params.clientId; | ||
this.provider = params.provider; | ||
this.username = params.username; | ||
this.password = params.password; | ||
this.passwordSecretName = params.passwordSecretName; | ||
this.token = params.token; | ||
} | ||
/** | ||
* Get the CMR password, from the AWS secret if set, else return the password | ||
* @returns {Promise.<string>} - the CMR password | ||
*/ | ||
getCmrPassword() { | ||
if (this.passwordSecretName) { | ||
return secretsManagerUtils.getSecretString( | ||
this.passwordSecretName | ||
); | ||
/** | ||
* The constructor for the CMR class | ||
* | ||
* @param {Object} params | ||
* @param {string} params.provider - the CMR provider id | ||
* @param {string} params.clientId - the CMR clientId | ||
* @param {string} params.username - CMR username, not used if token is provided | ||
* @param {string} params.passwordSecretName - CMR password secret, not used if token is provided | ||
* @param {string} params.password - CMR password, not used if token or | ||
* passwordSecretName is provided | ||
* @param {string} params.token - CMR or Launchpad token, | ||
* if not provided, CMR username and password are used to get a cmr token | ||
*/ | ||
constructor(params = {}) { | ||
this.clientId = params.clientId; | ||
this.provider = params.provider; | ||
this.username = params.username; | ||
this.password = params.password; | ||
this.passwordSecretName = params.passwordSecretName; | ||
this.token = params.token; | ||
} | ||
return this.password; | ||
} | ||
/** | ||
* The method for getting the token | ||
* | ||
* @returns {Promise.<string>} the token | ||
*/ | ||
async getToken() { | ||
return (this.token) ? this.token | ||
: updateToken(this.provider, this.clientId, this.username, await this.getCmrPassword()); | ||
} | ||
/** | ||
* Return object containing CMR request headers for PUT / POST / DELETE | ||
* | ||
* @param {Object} params | ||
* @param {string} [params.token] - CMR request token | ||
* @param {string} [params.ummgVersion] - UMMG metadata version string or null if echo10 metadata | ||
* @returns {Object} CMR headers object | ||
*/ | ||
getWriteHeaders(params = {}) { | ||
const contentType = params.ummgVersion | ||
? `application/vnd.nasa.cmr.umm+json;version=${params.ummgVersion}` | ||
: 'application/echo10+xml'; | ||
const headers = { | ||
'Client-Id': this.clientId, | ||
'Content-type': contentType | ||
}; | ||
if (params.token) headers['Echo-Token'] = params.token; | ||
if (params.ummgVersion) headers.Accept = 'application/json'; | ||
return headers; | ||
} | ||
/** | ||
* Return object containing CMR request headers for GETs | ||
* | ||
* @param {Object} params | ||
* @param {string} [params.token] - CMR request token | ||
* @returns {Object} CMR headers object | ||
*/ | ||
getReadHeaders(params = {}) { | ||
const headers = { | ||
'Client-Id': this.clientId | ||
}; | ||
if (params.token) headers['Echo-Token'] = params.token; | ||
return headers; | ||
} | ||
/** | ||
* Adds a collection record to the CMR | ||
* | ||
* @param {string} xml - the collection XML document | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async ingestCollection(xml) { | ||
const headers = this.getWriteHeaders({ token: await this.getToken() }); | ||
return ingestConcept('collection', xml, 'Collection.DataSetId', this.provider, headers); | ||
} | ||
/** | ||
* Adds a granule record to the CMR | ||
* | ||
* @param {string} xml - the granule XML document | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async ingestGranule(xml) { | ||
const headers = this.getWriteHeaders({ token: await this.getToken() }); | ||
return ingestConcept('granule', xml, 'Granule.GranuleUR', this.provider, headers); | ||
} | ||
/** | ||
* Adds/Updates UMMG json metadata in the CMR | ||
* | ||
* @param {Object} ummgMetadata - UMMG metadata object | ||
* @returns {Promise<Object>} to the CMR response object. | ||
*/ | ||
async ingestUMMGranule(ummgMetadata) { | ||
const headers = this.getWriteHeaders({ | ||
token: await this.getToken(), | ||
ummgVersion: ummVersion(ummgMetadata) | ||
}); | ||
const granuleId = ummgMetadata.GranuleUR || 'no GranuleId found on input metadata'; | ||
logDetails.granuleId = granuleId; | ||
let response; | ||
try { | ||
await validateUMMG(ummgMetadata, granuleId, this.provider); | ||
response = await got.put( | ||
`${getUrl('ingest', this.provider)}granules/${granuleId}`, | ||
{ | ||
json: true, | ||
body: ummgMetadata, | ||
headers | ||
/** | ||
* Get the CMR password, from the AWS secret if set, else return the password | ||
* @returns {Promise.<string>} - the CMR password | ||
*/ | ||
getCmrPassword() { | ||
if (this.passwordSecretName) { | ||
return secretsManagerUtils.getSecretString(this.passwordSecretName); | ||
} | ||
); | ||
if (response.body.errors) { | ||
throw new Error(`Failed to ingest, CMR Errors: ${response.errors}`); | ||
} | ||
} catch (error) { | ||
log.error(error, logDetails); | ||
throw error; | ||
return this.password; | ||
} | ||
return response.body; | ||
} | ||
/** | ||
* Deletes a collection record from the CMR | ||
* | ||
* @param {string} datasetID - the collection unique id | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async deleteCollection(datasetID) { | ||
const headers = this.getWriteHeaders({ token: await this.getToken() }); | ||
return deleteConcept('collection', datasetID, headers); | ||
} | ||
/** | ||
* Deletes a granule record from the CMR | ||
* | ||
* @param {string} granuleUR - the granule unique id | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async deleteGranule(granuleUR) { | ||
const headers = this.getWriteHeaders({ token: await this.getToken() }); | ||
return deleteConcept('granules', granuleUR, this.provider, headers); | ||
} | ||
async searchConcept(type, searchParams, format = 'json', recursive = true) { | ||
const headers = this.getReadHeaders({ token: await this.getToken() }); | ||
return searchConcept({ | ||
type, | ||
searchParams, | ||
previousResults: [], | ||
headers, | ||
format, | ||
recursive | ||
}); | ||
} | ||
/** | ||
* Search in collections | ||
* | ||
* @param {string} params - the search parameters | ||
* @param {string} [format=json] - format of the response | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async searchCollections(params, format = 'json') { | ||
const searchParams = { provider_short_name: this.provider, ...params }; | ||
return this.searchConcept( | ||
'collections', | ||
searchParams, | ||
format | ||
); | ||
} | ||
/** | ||
* Search in granules | ||
* | ||
* @param {string} params - the search parameters | ||
* @param {string} [format='json'] - format of the response | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async searchGranules(params, format = 'json') { | ||
const searchParams = { provider_short_name: this.provider, ...params }; | ||
return this.searchConcept( | ||
'granules', | ||
searchParams, | ||
format | ||
); | ||
} | ||
/** | ||
* Get the granule metadata from CMR using the cmrLink | ||
* | ||
* @param {string} cmrLink - URL to concept | ||
* @returns {Object} - metadata as a JS object, null if not found | ||
*/ | ||
async getGranuleMetadata(cmrLink) { | ||
const headers = this.getReadHeaders({ token: await this.getToken() }); | ||
return getConcept(cmrLink, headers); | ||
} | ||
/** | ||
* The method for getting the token | ||
* | ||
* @returns {Promise.<string>} the token | ||
*/ | ||
async getToken() { | ||
return (this.token) ? this.token | ||
: updateToken(this.provider, this.clientId, this.username, await this.getCmrPassword()); | ||
} | ||
/** | ||
* Return object containing CMR request headers for PUT / POST / DELETE | ||
* | ||
* @param {Object} params | ||
* @param {string} [params.token] - CMR request token | ||
* @param {string} [params.ummgVersion] - UMMG metadata version string or null if echo10 metadata | ||
* @returns {Object} CMR headers object | ||
*/ | ||
getWriteHeaders(params = {}) { | ||
const contentType = params.ummgVersion | ||
? `application/vnd.nasa.cmr.umm+json;version=${params.ummgVersion}` | ||
: 'application/echo10+xml'; | ||
const headers = { | ||
'Client-Id': this.clientId, | ||
'Content-type': contentType | ||
}; | ||
if (params.token) | ||
headers['Echo-Token'] = params.token; | ||
if (params.ummgVersion) | ||
headers.Accept = 'application/json'; | ||
return headers; | ||
} | ||
/** | ||
* Return object containing CMR request headers for GETs | ||
* | ||
* @param {Object} params | ||
* @param {string} [params.token] - CMR request token | ||
* @returns {Object} CMR headers object | ||
*/ | ||
getReadHeaders(params = {}) { | ||
const headers = { | ||
'Client-Id': this.clientId | ||
}; | ||
if (params.token) | ||
headers['Echo-Token'] = params.token; | ||
return headers; | ||
} | ||
/** | ||
* Adds a collection record to the CMR | ||
* | ||
* @param {string} xml - the collection XML document | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async ingestCollection(xml) { | ||
const headers = this.getWriteHeaders({ token: await this.getToken() }); | ||
return ingestConcept('collection', xml, 'Collection.DataSetId', this.provider, headers); | ||
} | ||
/** | ||
* Adds a granule record to the CMR | ||
* | ||
* @param {string} xml - the granule XML document | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async ingestGranule(xml) { | ||
const headers = this.getWriteHeaders({ token: await this.getToken() }); | ||
return ingestConcept('granule', xml, 'Granule.GranuleUR', this.provider, headers); | ||
} | ||
/** | ||
* Adds/Updates UMMG json metadata in the CMR | ||
* | ||
* @param {Object} ummgMetadata - UMMG metadata object | ||
* @returns {Promise<Object>} to the CMR response object. | ||
*/ | ||
async ingestUMMGranule(ummgMetadata) { | ||
const headers = this.getWriteHeaders({ | ||
token: await this.getToken(), | ||
ummgVersion: ummVersion(ummgMetadata) | ||
}); | ||
const granuleId = ummgMetadata.GranuleUR || 'no GranuleId found on input metadata'; | ||
logDetails.granuleId = granuleId; | ||
let response; | ||
try { | ||
await validateUMMG(ummgMetadata, granuleId, this.provider); | ||
response = await got.put(`${getUrl('ingest', this.provider)}granules/${granuleId}`, { | ||
json: true, | ||
body: ummgMetadata, | ||
headers | ||
}); | ||
if (response.body.errors) { | ||
throw new Error(`Failed to ingest, CMR Errors: ${response.errors}`); | ||
} | ||
} | ||
catch (error) { | ||
log.error(error, logDetails); | ||
throw error; | ||
} | ||
return response.body; | ||
} | ||
/** | ||
* Deletes a collection record from the CMR | ||
* | ||
* @param {string} datasetID - the collection unique id | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async deleteCollection(datasetID) { | ||
const headers = this.getWriteHeaders({ token: await this.getToken() }); | ||
return deleteConcept('collection', datasetID, headers); | ||
} | ||
/** | ||
* Deletes a granule record from the CMR | ||
* | ||
* @param {string} granuleUR - the granule unique id | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async deleteGranule(granuleUR) { | ||
const headers = this.getWriteHeaders({ token: await this.getToken() }); | ||
return deleteConcept('granules', granuleUR, this.provider, headers); | ||
} | ||
async searchConcept(type, searchParams, format = 'json', recursive = true) { | ||
const headers = this.getReadHeaders({ token: await this.getToken() }); | ||
return searchConcept({ | ||
type, | ||
searchParams, | ||
previousResults: [], | ||
headers, | ||
format, | ||
recursive | ||
}); | ||
} | ||
/** | ||
* Search in collections | ||
* | ||
* @param {string} params - the search parameters | ||
* @param {string} [format=json] - format of the response | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async searchCollections(params, format = 'json') { | ||
const searchParams = { provider_short_name: this.provider, ...params }; | ||
return this.searchConcept('collections', searchParams, format); | ||
} | ||
/** | ||
* Search in granules | ||
* | ||
* @param {string} params - the search parameters | ||
* @param {string} [format='json'] - format of the response | ||
* @returns {Promise.<Object>} the CMR response | ||
*/ | ||
async searchGranules(params, format = 'json') { | ||
const searchParams = { provider_short_name: this.provider, ...params }; | ||
return this.searchConcept('granules', searchParams, format); | ||
} | ||
/** | ||
* Get the granule metadata from CMR using the cmrLink | ||
* | ||
* @param {string} cmrLink - URL to concept | ||
* @returns {Object} - metadata as a JS object, null if not found | ||
*/ | ||
async getGranuleMetadata(cmrLink) { | ||
const headers = this.getReadHeaders({ token: await this.getToken() }); | ||
return getConcept(cmrLink, headers); | ||
} | ||
} | ||
module.exports = CMR; | ||
//# sourceMappingURL=CMR.js.map |
'use strict'; | ||
const CMR = require('./CMR'); | ||
/** | ||
@@ -23,65 +21,59 @@ * A class to efficiently list all of the concepts (collections/granules) from | ||
class CMRSearchConceptQueue { | ||
/** | ||
* The constructor for the CMRSearchConceptQueue class | ||
* | ||
* @param {Object} params | ||
* @param {string} params.cmrSettings - the CMR settings for the requests - the provider, | ||
* clientId, and either launchpad token or EDL username and password | ||
* @param {string} params.type - the type of search 'granule' or 'collection' | ||
* @param {string} [params.searchParams={}] - the search parameters | ||
* @param {string} params.format - the result format | ||
*/ | ||
constructor(params = { searchParams: {} }) { | ||
this.type = params.type; | ||
this.params = { provider_short_name: params.cmrSettings.provider, ...params.searchParams }; | ||
this.format = params.format; | ||
this.items = []; | ||
this.CMR = new CMR(params.cmrSettings); | ||
} | ||
/** | ||
* View the next item in the queue | ||
* | ||
* This does not remove the object from the queue. When there are no more | ||
* items in the queue, returns 'null'. | ||
* | ||
* @returns {Promise<Object>} an item from the CMR search | ||
*/ | ||
async peek() { | ||
if (this.items.length === 0) await this.fetchItems(); | ||
return this.items[0]; | ||
} | ||
/** | ||
* Remove the next item from the queue | ||
* | ||
* When there are no more items in the queue, returns `null`. | ||
* | ||
* @returns {Promise<Object>} an item from the CMR search | ||
*/ | ||
async shift() { | ||
if (this.items.length === 0) await this.fetchItems(); | ||
return this.items.shift(); | ||
} | ||
/** | ||
* Query the CMR API to get the next batch of items | ||
* | ||
* @returns {Promise<undefined>} resolves when the queue has been updated | ||
* @private | ||
*/ | ||
async fetchItems() { | ||
const results = await this.CMR.searchConcept( | ||
this.type, | ||
this.params, | ||
this.format, | ||
false | ||
); | ||
this.items = results; | ||
this.params.page_num = (this.params.page_num) ? this.params.page_num + 1 : 1; | ||
if (results.length === 0) this.items.push(null); | ||
} | ||
/** | ||
* The constructor for the CMRSearchConceptQueue class | ||
* | ||
* @param {Object} params | ||
* @param {string} params.cmrSettings - the CMR settings for the requests - the provider, | ||
* clientId, and either launchpad token or EDL username and password | ||
* @param {string} params.type - the type of search 'granule' or 'collection' | ||
* @param {string} [params.searchParams={}] - the search parameters | ||
* @param {string} params.format - the result format | ||
*/ | ||
constructor(params = { searchParams: {} }) { | ||
this.type = params.type; | ||
this.params = { provider_short_name: params.cmrSettings.provider, ...params.searchParams }; | ||
this.format = params.format; | ||
this.items = []; | ||
this.CMR = new CMR(params.cmrSettings); | ||
} | ||
/** | ||
* View the next item in the queue | ||
* | ||
* This does not remove the object from the queue. When there are no more | ||
* items in the queue, returns 'null'. | ||
* | ||
* @returns {Promise<Object>} an item from the CMR search | ||
*/ | ||
async peek() { | ||
if (this.items.length === 0) | ||
await this.fetchItems(); | ||
return this.items[0]; | ||
} | ||
/** | ||
* Remove the next item from the queue | ||
* | ||
* When there are no more items in the queue, returns `null`. | ||
* | ||
* @returns {Promise<Object>} an item from the CMR search | ||
*/ | ||
async shift() { | ||
if (this.items.length === 0) | ||
await this.fetchItems(); | ||
return this.items.shift(); | ||
} | ||
/** | ||
* Query the CMR API to get the next batch of items | ||
* | ||
* @returns {Promise<undefined>} resolves when the queue has been updated | ||
* @private | ||
*/ | ||
async fetchItems() { | ||
const results = await this.CMR.searchConcept(this.type, this.params, this.format, false); | ||
this.items = results; | ||
this.params.page_num = (this.params.page_num) ? this.params.page_num + 1 : 1; | ||
if (results.length === 0) | ||
this.items.push(null); | ||
} | ||
} | ||
module.exports = CMRSearchConceptQueue; | ||
//# sourceMappingURL=CMRSearchConceptQueue.js.map |
'use strict'; | ||
const Logger = require('@cumulus/logger'); | ||
const got = require('got'); | ||
const { parseXMLString } = require('./Utils'); | ||
const getUrl = require('./getUrl'); | ||
const log = new Logger({ sender: 'cmr-client' }); | ||
/** | ||
@@ -21,31 +17,28 @@ * Deletes a record from the CMR | ||
async function deleteConcept(type, identifier, provider, headers) { | ||
const url = `${getUrl('ingest', provider)}${type}/${identifier}`; | ||
log.info(`deleteConcept ${url}`); | ||
let result; | ||
try { | ||
result = await got.delete(url, { | ||
headers | ||
}); | ||
} catch (error) { | ||
result = error.response; | ||
} | ||
const xmlObject = await parseXMLString(result.body); | ||
let errorMessage; | ||
if (result.statusCode !== 200) { | ||
errorMessage = `Failed to delete, statusCode: ${result.statusCode}, statusMessage: ${result.statusMessage}`; | ||
if (xmlObject.errors) { | ||
errorMessage = `${errorMessage}, CMR error message: ${JSON.stringify(xmlObject.errors.error)}`; | ||
const url = `${getUrl('ingest', provider)}${type}/${identifier}`; | ||
log.info(`deleteConcept ${url}`); | ||
let result; | ||
try { | ||
result = await got.delete(url, { | ||
headers | ||
}); | ||
} | ||
log.info(errorMessage); | ||
} | ||
if (result.statusCode !== 200 && result.statusCode !== 404) { | ||
throw new Error(errorMessage); | ||
} | ||
return xmlObject; | ||
catch (error) { | ||
result = error.response; | ||
} | ||
const xmlObject = await parseXMLString(result.body); | ||
let errorMessage; | ||
if (result.statusCode !== 200) { | ||
errorMessage = `Failed to delete, statusCode: ${result.statusCode}, statusMessage: ${result.statusMessage}`; | ||
if (xmlObject.errors) { | ||
errorMessage = `${errorMessage}, CMR error message: ${JSON.stringify(xmlObject.errors.error)}`; | ||
} | ||
log.info(errorMessage); | ||
} | ||
if (result.statusCode !== 200 && result.statusCode !== 404) { | ||
throw new Error(errorMessage); | ||
} | ||
return xmlObject; | ||
} | ||
module.exports = deleteConcept; | ||
//# sourceMappingURL=deleteConcept.js.map |
'use strict'; | ||
const got = require('got'); | ||
const Logger = require('@cumulus/logger'); | ||
const log = new Logger({ sender: 'cmr-client' }); | ||
/** | ||
@@ -17,21 +14,18 @@ * Get the CMR JSON metadata from the cmrLink | ||
async function getConceptMetadata(conceptLink, headers) { | ||
let response; | ||
try { | ||
response = await got.get(conceptLink, { headers }); | ||
} catch (e) { | ||
log.error(`Error getting concept metadata from ${conceptLink}`, e); | ||
return null; | ||
} | ||
if (response.statusCode !== 200) { | ||
log.error(`Received statusCode ${response.statusCode} getting concept metadata from ${conceptLink}`); | ||
return null; | ||
} | ||
const body = JSON.parse(response.body); | ||
return body.feed.entry[0]; | ||
let response; | ||
try { | ||
response = await got.get(conceptLink, { headers }); | ||
} | ||
catch (error) { | ||
log.error(`Error getting concept metadata from ${conceptLink}`, error); | ||
return null; | ||
} | ||
if (response.statusCode !== 200) { | ||
log.error(`Received statusCode ${response.statusCode} getting concept metadata from ${conceptLink}`); | ||
return null; | ||
} | ||
const body = JSON.parse(response.body); | ||
return body.feed.entry[0]; | ||
} | ||
module.exports = getConceptMetadata; | ||
//# sourceMappingURL=getConcept.js.map |
'use strict'; | ||
const get = require('lodash/get'); | ||
/** | ||
@@ -11,9 +9,4 @@ * Returns the environment specific identifier for the input cmr environment. | ||
function hostId(env) { | ||
return get( | ||
{ OPS: '', SIT: 'sit', UAT: 'uat' }, | ||
env, | ||
'sit' | ||
); | ||
return get({ OPS: '', SIT: 'sit', UAT: 'uat' }, env, 'sit'); | ||
} | ||
/** | ||
@@ -32,8 +25,7 @@ * Determines the appropriate CMR host endpoint based on a given | ||
function getHost(cmrEnvironment, cmrHost) { | ||
if (cmrHost) return cmrHost; | ||
const host = ['cmr', hostId(cmrEnvironment), 'earthdata.nasa.gov'].filter((d) => d).join('.'); | ||
return host; | ||
if (cmrHost) | ||
return cmrHost; | ||
const host = ['cmr', hostId(cmrEnvironment), 'earthdata.nasa.gov'].filter((d) => d).join('.'); | ||
return host; | ||
} | ||
/** | ||
@@ -51,31 +43,30 @@ * returns the full url for various cmr services | ||
function getUrl(type, cmrProvider, cmrEnvironment, cmrHost) { | ||
let url; | ||
const cmrEnv = cmrEnvironment || process.env.CMR_ENVIRONMENT || null; | ||
const host = getHost(cmrEnv, cmrHost); | ||
const provider = cmrProvider; | ||
switch (type) { | ||
case 'token': | ||
if (cmrEnv === 'OPS') { | ||
url = 'https://cmr.earthdata.nasa.gov/legacy-services/rest/tokens'; | ||
} else { | ||
url = 'https://cmr.uat.earthdata.nasa.gov/legacy-services/rest/tokens'; | ||
let url; | ||
const cmrEnv = cmrEnvironment || process.env.CMR_ENVIRONMENT || null; | ||
const host = getHost(cmrEnv, cmrHost); | ||
const provider = cmrProvider; | ||
switch (type) { | ||
case 'token': | ||
if (cmrEnv === 'OPS') { | ||
url = 'https://cmr.earthdata.nasa.gov/legacy-services/rest/tokens'; | ||
} | ||
else { | ||
url = 'https://cmr.uat.earthdata.nasa.gov/legacy-services/rest/tokens'; | ||
} | ||
break; | ||
case 'search': | ||
url = `https://${host}/search/`; | ||
break; | ||
case 'validate': | ||
url = `https://${host}/ingest/providers/${provider}/validate/`; | ||
break; | ||
case 'ingest': | ||
url = `https://${host}/ingest/providers/${provider}/`; | ||
break; | ||
default: | ||
url = null; | ||
} | ||
break; | ||
case 'search': | ||
url = `https://${host}/search/`; | ||
break; | ||
case 'validate': | ||
url = `https://${host}/ingest/providers/${provider}/validate/`; | ||
break; | ||
case 'ingest': | ||
url = `https://${host}/ingest/providers/${provider}/`; | ||
break; | ||
default: | ||
url = null; | ||
} | ||
return url; | ||
return url; | ||
} | ||
module.exports = getUrl; | ||
//# sourceMappingURL=getUrl.js.map |
'use strict'; | ||
const CMR = require('./CMR'); | ||
const CMRSearchConceptQueue = require('./CMRSearchConceptQueue'); | ||
const ValidationError = require('./ValidationError'); | ||
module.exports = { | ||
CMR, | ||
CMRSearchConceptQueue, | ||
ValidationError | ||
CMR, | ||
CMRSearchConceptQueue, | ||
ValidationError | ||
}; | ||
//# sourceMappingURL=index.js.map |
'use strict'; | ||
const got = require('got'); | ||
const property = require('lodash/property'); | ||
const Logger = require('@cumulus/logger'); | ||
const validate = require('./validate'); | ||
const getUrl = require('./getUrl'); | ||
const { parseXMLString } = require('./Utils'); | ||
const log = new Logger({ sender: 'cmr-client' }); | ||
const logDetails = { | ||
file: 'cmr-client/ingestConcept.js' | ||
file: 'cmr-client/ingestConcept.js' | ||
}; | ||
/** | ||
@@ -29,31 +24,24 @@ * Posts a record of any kind (collection, granule, etc) to | ||
async function ingestConcept(type, xmlString, identifierPath, provider, headers) { | ||
let xmlObject = await parseXMLString(xmlString); | ||
const identifier = property(identifierPath)(xmlObject); | ||
logDetails.granuleId = identifier; | ||
try { | ||
await validate(type, xmlString, identifier, provider); | ||
const response = await got.put( | ||
`${getUrl('ingest', provider)}${type}s/${identifier}`, | ||
{ | ||
body: xmlString, | ||
headers | ||
} | ||
); | ||
xmlObject = await parseXMLString(response.body); | ||
if (xmlObject.errors) { | ||
const xmlObjectError = JSON.stringify(xmlObject.errors.error); | ||
throw new Error(`Failed to ingest, CMR error message: ${xmlObjectError}`); | ||
let xmlObject = await parseXMLString(xmlString); | ||
const identifier = property(identifierPath)(xmlObject); | ||
logDetails.granuleId = identifier; | ||
try { | ||
await validate(type, xmlString, identifier, provider); | ||
const response = await got.put(`${getUrl('ingest', provider)}${type}s/${identifier}`, { | ||
body: xmlString, | ||
headers | ||
}); | ||
xmlObject = await parseXMLString(response.body); | ||
if (xmlObject.errors) { | ||
const xmlObjectError = JSON.stringify(xmlObject.errors.error); | ||
throw new Error(`Failed to ingest, CMR error message: ${xmlObjectError}`); | ||
} | ||
return xmlObject; | ||
} | ||
return xmlObject; | ||
} catch (e) { | ||
log.error(e, logDetails); | ||
throw e; | ||
} | ||
catch (error) { | ||
log.error(error, logDetails); | ||
throw error; | ||
} | ||
} | ||
module.exports = ingestConcept; | ||
//# sourceMappingURL=ingestConcept.js.map |
{ | ||
"name": "@cumulus/cmr-client", | ||
"version": "1.24.0", | ||
"version": "2.0.0", | ||
"description": "A Node.js client to NASA's Common Metadata Repository (CMR) API.", | ||
"engines": { | ||
"node": ">=10.16.3" | ||
"node": ">=12.18.0" | ||
}, | ||
"scripts": { | ||
"build-docs": "../../node_modules/.bin/jsdoc2md --heading-depth 2 --template templates/API.hbs CMR.js CMRSearchConceptQueue.js > API.md", | ||
"clean": "rm -f ./*.js ./*.d.ts", | ||
"prepare": "npm run tsc", | ||
"test": "../../node_modules/.bin/ava", | ||
"test-coverage": "../../node_modules/.bin/nyc npm test", | ||
"debug": "NODE_ENV=test node --inspect-brk node_modules/ava/profile.js --serial tests/*.js" | ||
"test:coverage": "../../node_modules/.bin/nyc npm test", | ||
"tsc": "../../node_modules/.bin/tsc", | ||
"watch-test": "../../node_modules/.bin/tsc-watch --onsuccess 'npm test'" | ||
}, | ||
@@ -17,7 +21,2 @@ "ava": { | ||
}, | ||
"nyc": { | ||
"exclude": [ | ||
"tests" | ||
] | ||
}, | ||
"keywords": [ | ||
@@ -29,5 +28,7 @@ "CUMULUS" | ||
}, | ||
"homepage": "https://github.com/nasa/cumulus/tree/master/packages/cmr-client#readme", | ||
"repository": { | ||
"type": "git", | ||
"url": "https://github.com/nasa/cumulus" | ||
"url": "https://github.com/nasa/cumulus", | ||
"directory": "packages/cmr-client" | ||
}, | ||
@@ -37,4 +38,4 @@ "author": "Cumulus Authors", | ||
"dependencies": { | ||
"@cumulus/aws-client": "1.24.0", | ||
"@cumulus/logger": "1.24.0", | ||
"@cumulus/aws-client": "2.0.0", | ||
"@cumulus/logger": "2.0.0", | ||
"got": "^9.6.0", | ||
@@ -45,3 +46,3 @@ "lodash": "^4.17.15", | ||
}, | ||
"gitHead": "e98bd892450ac176e8e802a69d2c7d27a9ed3bca" | ||
"gitHead": "404fd959be4f17ccdf4f047f591ada7c2e39e3e9" | ||
} |
'use strict'; | ||
const got = require('got'); | ||
const getUrl = require('./getUrl'); | ||
const { parseXMLString } = require('./Utils'); | ||
/** | ||
@@ -24,48 +22,31 @@ * | ||
*/ | ||
async function searchConcept({ | ||
type, | ||
searchParams, | ||
previousResults = [], | ||
headers = {}, | ||
format = 'json', | ||
recursive = true, | ||
cmrEnvironment = process.env.CMR_ENVIRONMENT, | ||
cmrLimit = process.env.CMR_LIMIT, | ||
cmrPageSize = process.env.CMR_PAGE_SIZE | ||
}) { | ||
const recordsLimit = cmrLimit || 100; | ||
const pageSize = searchParams.pageSize || cmrPageSize || 50; | ||
const defaultParams = { page_size: pageSize }; | ||
const url = `${getUrl('search', null, cmrEnvironment)}${type}.${format.toLowerCase()}`; | ||
const pageNum = (searchParams.page_num) ? searchParams.page_num + 1 : 1; | ||
// if requested, recursively retrieve all the search results for collections or granules | ||
const query = { ...defaultParams, ...searchParams, page_num: pageNum }; | ||
const response = await got.get(url, { json: format.endsWith('json'), query, headers }); | ||
const responseItems = (format === 'echo10') | ||
? (await parseXMLString(response.body)).results.result || [] | ||
: (response.body.items || response.body.feed.entry); | ||
const fetchedResults = previousResults.concat(responseItems || []); | ||
const numRecordsCollected = fetchedResults.length; | ||
const CMRHasMoreResults = response.headers['cmr-hits'] > numRecordsCollected; | ||
const recordsLimitReached = numRecordsCollected >= recordsLimit; | ||
if (recursive && CMRHasMoreResults && !recordsLimitReached) { | ||
return searchConcept({ | ||
type, | ||
searchParams: query, | ||
previousResults: fetchedResults, | ||
headers, | ||
format, | ||
recursive | ||
}); | ||
} | ||
return fetchedResults.slice(0, recordsLimit); | ||
async function searchConcept({ type, searchParams, previousResults = [], headers = {}, format = 'json', recursive = true, cmrEnvironment = process.env.CMR_ENVIRONMENT, cmrLimit = process.env.CMR_LIMIT, cmrPageSize = process.env.CMR_PAGE_SIZE }) { | ||
const recordsLimit = cmrLimit || 100; | ||
const pageSize = searchParams.pageSize || cmrPageSize || 50; | ||
const defaultParams = { page_size: pageSize }; | ||
const url = `${getUrl('search', null, cmrEnvironment)}${type}.${format.toLowerCase()}`; | ||
const pageNum = (searchParams.page_num) ? searchParams.page_num + 1 : 1; | ||
// if requested, recursively retrieve all the search results for collections or granules | ||
const query = { ...defaultParams, ...searchParams, page_num: pageNum }; | ||
const response = await got.get(url, { json: format.endsWith('json'), query, headers }); | ||
const responseItems = (format === 'echo10') | ||
? (await parseXMLString(response.body)).results.result || [] | ||
: (response.body.items || response.body.feed.entry); | ||
const fetchedResults = previousResults.concat(responseItems || []); | ||
const numRecordsCollected = fetchedResults.length; | ||
const CMRHasMoreResults = response.headers['cmr-hits'] > numRecordsCollected; | ||
const recordsLimitReached = numRecordsCollected >= recordsLimit; | ||
if (recursive && CMRHasMoreResults && !recordsLimitReached) { | ||
return searchConcept({ | ||
type, | ||
searchParams: query, | ||
previousResults: fetchedResults, | ||
headers, | ||
format, | ||
recursive | ||
}); | ||
} | ||
return fetchedResults.slice(0, recordsLimit); | ||
} | ||
module.exports = searchConcept; | ||
//# sourceMappingURL=searchConcept.js.map |
'use strict'; | ||
const get = require('lodash/get'); | ||
@@ -7,3 +6,2 @@ const got = require('got'); | ||
const ValidationError = require('./ValidationError'); | ||
/** | ||
@@ -18,3 +16,2 @@ * Find the UMM version as a decimal string. | ||
const ummVersion = (umm) => get(umm, 'MetadataSpecification.Version', '1.4'); | ||
/** | ||
@@ -30,25 +27,20 @@ * Posts a given XML string to the validate endpoint of CMR and throws an | ||
const validateUMMG = async (ummMetadata, identifier, provider) => { | ||
const version = ummVersion(ummMetadata); | ||
const { statusCode, body } = await got.post( | ||
`${getUrl('validate', provider)}granule/${identifier}`, | ||
{ | ||
json: true, | ||
body: ummMetadata, | ||
headers: { | ||
Accept: 'application/json', | ||
'Content-type': `application/vnd.nasa.cmr.umm+json;version=${version}` | ||
}, | ||
throwHttpErrors: false | ||
} | ||
); | ||
if (statusCode === 200) return; | ||
throw new ValidationError(`Validation was not successful, CMR error message: ${JSON.stringify(body.errors)}`); | ||
const version = ummVersion(ummMetadata); | ||
const { statusCode, body } = await got.post(`${getUrl('validate', provider)}granule/${identifier}`, { | ||
json: true, | ||
body: ummMetadata, | ||
headers: { | ||
Accept: 'application/json', | ||
'Content-type': `application/vnd.nasa.cmr.umm+json;version=${version}` | ||
}, | ||
throwHttpErrors: false | ||
}); | ||
if (statusCode === 200) | ||
return; | ||
throw new ValidationError(`Validation was not successful, CMR error message: ${JSON.stringify(body.errors)}`); | ||
}; | ||
module.exports = { | ||
ummVersion, | ||
validateUMMG | ||
ummVersion, | ||
validateUMMG | ||
}; | ||
//# sourceMappingURL=UmmUtils.js.map |
19
Utils.js
'use strict'; | ||
const { promisify } = require('util'); | ||
const xml2js = require('xml2js'); | ||
async function parseXMLString(xmlString) { | ||
const parseString = promisify(xml2js.parseString); | ||
const xmlParseOptions = { | ||
ignoreAttrs: true, | ||
mergeAttrs: true, | ||
explicitArray: false | ||
}; | ||
return parseString(xmlString, xmlParseOptions); | ||
const parseString = promisify(xml2js.parseString); | ||
const xmlParseOptions = { | ||
ignoreAttrs: true, | ||
mergeAttrs: true, | ||
explicitArray: false | ||
}; | ||
return parseString(xmlString, xmlParseOptions); | ||
} | ||
exports.parseXMLString = parseXMLString; | ||
//# sourceMappingURL=Utils.js.map |
'use strict'; | ||
const got = require('got'); | ||
@@ -7,3 +6,2 @@ const ValidationError = require('./ValidationError'); | ||
const { parseXMLString } = require('./Utils'); | ||
/** | ||
@@ -20,25 +18,21 @@ * Posts a given xml string to the validate endpoint of the CMR | ||
async function validate(type, xml, identifier, provider) { | ||
let result; | ||
try { | ||
result = await got.post(`${getUrl('validate', provider)}${type}/${identifier}`, { | ||
body: xml, | ||
headers: { | ||
'Content-type': 'application/echo10+xml' | ||
} | ||
}); | ||
if (result.statusCode === 200) { | ||
return true; | ||
let result; | ||
try { | ||
result = await got.post(`${getUrl('validate', provider)}${type}/${identifier}`, { | ||
body: xml, | ||
headers: { | ||
'Content-type': 'application/echo10+xml' | ||
} | ||
}); | ||
if (result.statusCode === 200) { | ||
return true; | ||
} | ||
} | ||
} catch (e) { | ||
result = e.response; | ||
} | ||
const parsed = await parseXMLString(result.body); | ||
throw new ValidationError( | ||
`Validation was not successful, CMR error message: ${JSON.stringify(parsed.errors.error)}` | ||
); | ||
catch (error) { | ||
result = error.response; | ||
} | ||
const parsed = await parseXMLString(result.body); | ||
throw new ValidationError(`Validation was not successful, CMR error message: ${JSON.stringify(parsed.errors.error)}`); | ||
} | ||
module.exports = validate; | ||
//# sourceMappingURL=validate.js.map |
'use strict'; | ||
class ValidationError extends Error { | ||
constructor(message) { | ||
super(message); | ||
this.name = this.constructor.name; | ||
} | ||
constructor(message) { | ||
super(message); | ||
this.name = this.constructor.name; | ||
} | ||
} | ||
module.exports = ValidationError; | ||
//# sourceMappingURL=ValidationError.js.map |
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
No website
QualityPackage does not have a website.
Found 1 instance in 1 package
104265
66
1753
0
8
1
+ Added@cumulus/aws-client@2.0.0(transitive)
+ Added@cumulus/checksum@2.0.0(transitive)
+ Added@cumulus/errors@2.0.0(transitive)
+ Added@cumulus/logger@2.0.0(transitive)
- Removed@cumulus/aws-client@1.24.0(transitive)
- Removed@cumulus/checksum@1.24.0(transitive)
- Removed@cumulus/errors@1.24.0(transitive)
- Removed@cumulus/logger@1.24.0(transitive)
Updated@cumulus/aws-client@2.0.0
Updated@cumulus/logger@2.0.0