@financial-times/biz-ops-schema
Advanced tools
Comparing version 2.0.0-beta.8 to 2.0.0-beta.9
23
index.js
@@ -1,22 +0,3 @@ | ||
const type = require('./methods/get-type'); | ||
const types = require('./methods/get-types'); | ||
const enums = require('./methods/get-enums'); | ||
const graphqlDefs = require('./methods/get-graphql-defs'); | ||
const primitiveTypes = require('./lib/primitive-types-map'); | ||
const sendSchemaToS3 = require('./lib/send-schema-to-s3'); | ||
const poller = require('./lib/poller'); | ||
const validate = require('./lib/validate'); | ||
const { init } = require('./lib/get-instance'); | ||
module.exports = Object.assign( | ||
{ | ||
getType: type, | ||
getTypes: types, | ||
getEnums: enums, | ||
getGraphqlDefs: graphqlDefs, | ||
normalizeTypeName: name => name, | ||
primitiveTypesMap: primitiveTypes, | ||
sendSchemaToS3, | ||
poller, | ||
}, | ||
validate, | ||
); | ||
module.exports = init(); |
@@ -1,19 +0,26 @@ | ||
let cache = {}; | ||
const deepFreeze = require('deep-freeze'); | ||
const clear = () => { | ||
cache = {}; | ||
return cache; | ||
}; | ||
class Cache { | ||
constructor() { | ||
this.cache = {}; | ||
} | ||
module.exports = { | ||
clear, | ||
cacheify: (func, keyFunc) => (...args) => { | ||
const key = keyFunc(...args); | ||
if (key in cache) { | ||
return cache[key]; | ||
} | ||
const val = func(...args); | ||
cache[key] = val; | ||
return val; | ||
}, | ||
}; | ||
clear() { | ||
this.cache = {}; | ||
return this.cache; | ||
} | ||
addCacheToFunction(func, keyFunc) { | ||
return (...args) => { | ||
const key = keyFunc(...args); | ||
if (key in this.cache) { | ||
return this.cache[key]; | ||
} | ||
const val = func(...args); | ||
this.cache[key] = val ? deepFreeze(val) : null; | ||
return val; | ||
}; | ||
} | ||
} | ||
module.exports = Cache; |
const semver = require('semver'); | ||
const { version: libVersion } = require('../package.json'); | ||
const getSchemaFilename = version => { | ||
const getSchemaFilename = (version = libVersion) => { | ||
const majorVersion = semver.major(version); | ||
@@ -5,0 +6,0 @@ const isPrerelease = !!semver.prerelease(version); |
@@ -0,1 +1,2 @@ | ||
// TODO: make this a part of the schema that is fetched from s3 | ||
// Biz-ops field type is the key, graphql type is the value | ||
@@ -2,0 +3,0 @@ module.exports = { |
@@ -1,22 +0,147 @@ | ||
const { clear: clearCache } = require('./cache'); | ||
const EventEmitter = require('events'); | ||
const fetch = require('node-fetch'); | ||
const deepFreeze = require('deep-freeze'); | ||
const Cache = require('./cache'); | ||
const readYaml = require('./read-yaml'); | ||
const getSchemaFilename = require('./get-schema-filename'); | ||
const { version: libVersion } = require('../package.json'); | ||
let cachedData = { | ||
schema: { | ||
types: readYaml.directory('types'), | ||
stringPatterns: readYaml.file('string-patterns.yaml'), | ||
enums: readYaml.file('enums.yaml'), | ||
}, | ||
}; | ||
class RawData { | ||
constructor(options) { | ||
this.eventEmitter = new EventEmitter(); | ||
this.lastRefreshDate = 0; | ||
this.cache = new Cache(); | ||
this.configure(options); | ||
// TODO improve this | ||
// currently when creating new instance defaults to dev, so always tries to fetch rawData from | ||
// yaml before the app gets a change to call configure to take out of dev mode | ||
// need to think of a way to delay this. | ||
// Mayeb configure should be called init() instead | ||
// Maybe a static method getDevInstance() woudl be useful for tests | ||
// but then tests don't use the same code as src... hmmm | ||
try { | ||
this.rawData = deepFreeze({ | ||
schema: { | ||
types: readYaml.directory('types'), | ||
stringPatterns: readYaml.file('string-patterns.yaml'), | ||
enums: readYaml.file('enums.yaml'), | ||
}, | ||
}); | ||
} catch (e) { | ||
this.rawData = {}; | ||
} | ||
} | ||
module.exports = { | ||
getTypes: () => cachedData.schema.types, | ||
getStringPatterns: () => cachedData.schema.stringPatterns, | ||
getEnums: () => cachedData.schema.enums, | ||
getVersion: () => cachedData.version, | ||
getAll: () => cachedData, | ||
set: data => { | ||
cachedData = data; | ||
clearCache(); | ||
}, | ||
}; | ||
configure({ | ||
updateMode = 'dev', // also 'stale' or 'poll' | ||
ttl = 60000, | ||
baseUrl, | ||
logger = console, | ||
} = {}) { | ||
this.updateMode = updateMode; | ||
this.ttl = ttl; | ||
this.baseUrl = baseUrl; | ||
this.logger = logger; | ||
this.url = `${this.baseUrl}/${getSchemaFilename(libVersion)}`; | ||
} | ||
getTypes() { | ||
return this.rawData.schema.types; | ||
} | ||
getStringPatterns() { | ||
return this.rawData.schema.stringPatterns; | ||
} | ||
getEnums() { | ||
return this.rawData.schema.enums; | ||
} | ||
getVersion() { | ||
return this.rawData.version; | ||
} | ||
getAll() { | ||
return this.rawData; | ||
} | ||
setRawData(data) { | ||
this.rawData = deepFreeze(data); | ||
this.cache.clear(); | ||
} | ||
on(event, func) { | ||
return this.eventEmitter.on(event, func); | ||
} | ||
refresh() { | ||
if (this.updateMode === 'dev') { | ||
return Promise.resolve(); | ||
} | ||
if (this.updateMode !== 'stale') { | ||
throw new Error('Cannot refresh when updateMode is not "stale"'); | ||
} | ||
if (Date.now() - this.lastRefreshDate > this.ttl) { | ||
return this.fetch(); | ||
} | ||
return Promise.resolve(); | ||
} | ||
fetch() { | ||
this.lastRefreshDate = Date.now(); | ||
this.logger.info({ | ||
event: 'FETCHING_SCHEMA', | ||
url: this.url, | ||
}); | ||
return fetch(this.url) | ||
.then(res => res.json()) | ||
.then(data => { | ||
const oldVersion = this.getVersion(); | ||
if (data.version === oldVersion) { | ||
this.logger.debug({ event: 'SCHEMA_NOT_CHANGED' }); | ||
return; | ||
} | ||
this.setRawData(data); | ||
this.logger.info({ | ||
event: 'SCHEMA_UPDATED', | ||
newVersion: data.version, | ||
oldVersion, | ||
}); | ||
this.eventEmitter.emit('change', { | ||
newVersion: data.version, | ||
oldVersion, | ||
}); | ||
}) | ||
.catch(error => | ||
this.logger.error({ event: 'SCHEMA_UPDATE_FAILED', error }), | ||
); | ||
} | ||
startPolling() { | ||
if (this.updateMode === 'dev') { | ||
return Promise.resolve(); | ||
} | ||
if (this.updateMode !== 'poll') { | ||
throw new Error( | ||
'Cannot start polling when updateMode is not "poll"', | ||
); | ||
} | ||
if (this.firstFetch) { | ||
return this.firstFetch; | ||
} | ||
this.logger.info({ | ||
event: 'STARTING_SCHEMA_POLLER', | ||
}); | ||
this.timer = setInterval(() => this.fetch(), this.ttl).unref(); | ||
this.firstFetch = this.fetch(); | ||
return this.firstFetch; | ||
} | ||
stopPolling() { | ||
clearInterval(this.timer); | ||
delete this.firstFetch; | ||
} | ||
} | ||
module.exports = RawData; |
const AWS = require('aws-sdk'); | ||
const { Readable } = require('stream'); | ||
const getSchemaFilename = require('./get-schema-filename'); | ||
const rawData = require('./raw-data'); | ||
const s3Client = new AWS.S3({ region: 'eu-west-1' }); | ||
const sendSchemaToS3 = async (environment, schemaObject = rawData.getAll()) => { | ||
const sendSchemaToS3 = async (environment, schemaObject) => { | ||
const { version } = schemaObject; | ||
@@ -16,2 +15,9 @@ const schemaFilename = getSchemaFilename(version); | ||
uploadStream.push(null); | ||
console.log( | ||
`Deploying schema to biz-ops-schema.${ | ||
process.env.AWS_ACCOUNT_ID | ||
}/${environment}/${schemaFilename}`, | ||
); | ||
await s3Client | ||
@@ -18,0 +24,0 @@ .upload({ |
@@ -1,4 +0,1 @@ | ||
const getEnums = require('../methods/get-enums'); | ||
const getType = require('../methods/get-type'); | ||
const propertyNameRegex = /^[a-z][a-zA-Z\d]+$/; | ||
@@ -9,94 +6,86 @@ const primitiveTypesMap = require('./primitive-types-map'); | ||
class BizOpsError { | ||
constructor(message) { | ||
this.message = message; | ||
} | ||
} | ||
const BizOpsError = require('./biz-ops-error'); | ||
const validateTypeName = type => { | ||
if (!getType(type)) { | ||
throw new BizOpsError(`Invalid node type \`${type}\``); | ||
} | ||
}; | ||
const validateTypeName = getType => type => getType(type); | ||
const validateProperty = ( | ||
const throwInvalidValueError = ( | ||
typeName, | ||
propertyName, | ||
propertyValue, | ||
allowUnknown, // needed for v1 API.. for now | ||
) => { | ||
const propertyDefinition = getType(typeName).properties[propertyName]; | ||
) => reason => { | ||
throw new BizOpsError( | ||
`Invalid value \`${propertyValue}\` for property \`${propertyName}\` on type \`${typeName}\`. | ||
${reason}`, | ||
); | ||
}; | ||
if (!propertyDefinition) { | ||
if (allowUnknown) { | ||
return true; | ||
} | ||
throw new BizOpsError( | ||
`Invalid property \`${propertyName}\` on type \`${typeName}\`.`, | ||
); | ||
} | ||
const validateProperty = ({ getType, getEnums }) => { | ||
const recursivelyCallableValidator = ( | ||
typeName, | ||
propertyName, | ||
propertyValue, | ||
) => { | ||
const propertyDefinition = getType(typeName).properties[propertyName]; | ||
if (propertyValue === null) { | ||
return; | ||
} | ||
const { validator, relationship } = propertyDefinition; | ||
const type = | ||
primitiveTypesMap[propertyDefinition.type] || propertyDefinition.type; | ||
if (type === 'Boolean') { | ||
if (typeof propertyValue !== 'boolean') { | ||
if (!propertyDefinition) { | ||
throw new BizOpsError( | ||
`Invalid value \`${propertyValue}\` for property \`${propertyName}\` on type \`${typeName}\`. | ||
Must be a Boolean`, | ||
`Invalid property \`${propertyName}\` on type \`${typeName}\`.`, | ||
); | ||
} | ||
} else if (type === 'Float') { | ||
if (!Number.isFinite(propertyValue)) { | ||
throw new BizOpsError( | ||
`Invalid value \`${propertyValue}\` for property \`${propertyName}\` on type \`${typeName}\`. | ||
Must be a finite floating point number`, | ||
); | ||
if (propertyValue === null) { | ||
return; | ||
} | ||
} else if (type === 'Int') { | ||
if ( | ||
!Number.isFinite(propertyValue) || | ||
Math.round(propertyValue) !== propertyValue | ||
) { | ||
throw new BizOpsError( | ||
`Invalid value \`${propertyValue}\` for property \`${propertyName}\` on type \`${typeName}\`. | ||
Must be a finite integer`, | ||
const { validator, isRelationship } = propertyDefinition; | ||
const type = | ||
primitiveTypesMap[propertyDefinition.type] || | ||
propertyDefinition.type; | ||
const exit = throwInvalidValueError( | ||
typeName, | ||
propertyName, | ||
propertyValue, | ||
); | ||
if (isRelationship) { | ||
toArray(propertyValue).map(value => | ||
recursivelyCallableValidator(type, 'code', value), | ||
); | ||
} else if (type === 'Boolean') { | ||
if (typeof propertyValue !== 'boolean') { | ||
exit('Must be a Boolean'); | ||
} | ||
} else if (type === 'Float') { | ||
if (!Number.isFinite(propertyValue)) { | ||
exit('Must be a finite floating point number'); | ||
} | ||
} else if (type === 'Int') { | ||
if ( | ||
!Number.isFinite(propertyValue) || | ||
Math.round(propertyValue) !== propertyValue | ||
) { | ||
exit('Must be a finite integer'); | ||
} | ||
} else if (type === 'String') { | ||
if (typeof propertyValue !== 'string') { | ||
exit('Must be a string'); | ||
} | ||
if (validator && !validator.test(propertyValue)) { | ||
exit( | ||
`Must match pattern ${validator} and be no more than 64 characters`, | ||
); | ||
} | ||
} else if (type in getEnums()) { | ||
const validVals = Object.values(getEnums()[type]); | ||
if (!validVals.includes(propertyValue)) { | ||
exit(`Must be a valid enum: ${validVals.join(', ')}`); | ||
} | ||
} | ||
} else if (type === 'String') { | ||
if (typeof propertyValue !== 'string') { | ||
throw new BizOpsError( | ||
`Invalid value \`${propertyValue}\` for property \`${propertyName}\` on type \`${typeName}\`. | ||
Must be a string`, | ||
); | ||
} | ||
if (validator && !validator.test(propertyValue)) { | ||
throw new BizOpsError( | ||
`Invalid value \`${propertyValue}\` for property \`${propertyName}\` on type \`${typeName}\`. | ||
Must match pattern ${validator} and be no more than 64 characters`, | ||
); | ||
} | ||
} else if (type in getEnums()) { | ||
const validVals = Object.values(getEnums()[type]); | ||
if (!validVals.includes(propertyValue)) { | ||
throw new BizOpsError( | ||
`Invalid value \`${propertyValue}\` for property \`${propertyName}\` on type \`${typeName}\`. | ||
Must be a valid enum: ${validVals.join(', ')}`, | ||
); | ||
} | ||
// validate related codes | ||
} else if (relationship) { | ||
toArray(propertyValue).map(value => | ||
validateProperty(type, 'code', value), | ||
); | ||
} | ||
}; | ||
return recursivelyCallableValidator; | ||
}; | ||
const validateCode = (type, code) => validateProperty(type, 'code', code); | ||
const validatePropertyName = name => { | ||
@@ -114,8 +103,11 @@ // FIXME: allow SF_ID as, at least for a while, we need this to exist so that | ||
module.exports = { | ||
validateTypeName, | ||
validateProperty, | ||
validatePropertyName, | ||
validateCode, | ||
BizOpsError, | ||
module.exports = ({ getEnums, getType }) => { | ||
const propertyValidator = validateProperty({ getEnums, getType }); | ||
return { | ||
validateTypeName: validateTypeName(getType), | ||
validateProperty: propertyValidator, | ||
validatePropertyName, | ||
validateCode: (type, code) => propertyValidator(type, 'code', code), | ||
BizOpsError, | ||
}; | ||
}; |
{ | ||
"name": "@financial-times/biz-ops-schema", | ||
"version": "2.0.0-beta.8", | ||
"version": "2.0.0-beta.9", | ||
"description": "Schema for biz-ops data store and api. It provides two things: - yaml files which define which types, properties and relationships are allowed - a nodejs library for extracting subsets of this information", | ||
@@ -32,3 +32,2 @@ "main": "index.js", | ||
"@financial-times/n-logger": "^5.7.2", | ||
"semver": "^5.6.0", | ||
"clone": "^2.1.2", | ||
@@ -39,4 +38,5 @@ "common-tags": "^1.8.0", | ||
"lodash": "^4.17.10", | ||
"node-fetch": "^2.3.0" | ||
"node-fetch": "^2.3.0", | ||
"semver": "^5.6.0" | ||
} | ||
} |
@@ -12,11 +12,44 @@ # biz-ops-schema | ||
In production the component should be used in either 'poll' or 'stale' update modes, depending on the type of environment | ||
### Persistent nodejs process (e.g. heroku) | ||
```js | ||
const { poller } = require('@financial-times/biz-ops-schema'); | ||
poller.start(process.env.SCHEMA_BASE_URL); | ||
const { configure, startPolling } = require('@financial-times/biz-ops-schema'); | ||
configure({ | ||
baseUrl: process.env.SCHEMA_BASE_URL, | ||
updateMode: 'poll', | ||
logger: require('n-logger'), // or whichever logger you prefer | ||
ttl: 10000, // in milliseconds, defaults to 60000 | ||
}); | ||
startPolling().then(() => { | ||
// you can now start your app and use the schema | ||
}); | ||
``` | ||
### Transient nodejs process (e.g. AWS lambda) | ||
```js | ||
const { configure, refresh } = require('@financial-times/biz-ops-schema'); | ||
configure({ | ||
baseUrl: process.env.SCHEMA_BASE_URL, | ||
updateMode: 'stale', | ||
logger: require('n-lambda-logger'), // or whichever logger you prefer | ||
ttl: 10000, // in milliseconds, defaults to 60000 | ||
}); | ||
// in your function handler | ||
const handler = async event => { | ||
await refresh(); | ||
// now go ahead | ||
}; | ||
``` | ||
Speak to a member of the [biz ops team](https://financialtimes.slack.com/messages/C9S0V2KPV) to obtain a suitable value for `SCHEMA_BASE_URL`. | ||
The component _may_ be used without starting the poller - it will use a local copy of the schema provided as part of the npm package. However, unless there are specific reasons to want to pin to a specific schema version, it is far better to enable polling. | ||
### Local development | ||
When npm linking to test schema changes in an application, set `updateMode: 'dev'` to retrieve schema files from the local yaml files and disable polling/refersh on stale. | ||
## Adding to the schema | ||
@@ -57,2 +90,3 @@ | ||
- `groupProperties` [default: `false`]: Each property may have a `fieldset` attribute. Setting `groupProperties: true` removes the `properties` object from the data, and replaces it with `fieldsets`, where all properties are then grouped by fieldset | ||
- `includeMetaFields` [default: `false`]: Determines whether to include metadatafields (prefixed with `_`) in the schema object returned | ||
@@ -59,0 +93,0 @@ ### getTypes(options) |
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 1 instance in 1 package
136
3
2
31622
22
801