mongodb-schema
Advanced tools
Comparing version
@@ -1,17 +0,18 @@ | ||
var Schema = require('./schema'); | ||
var stream = require('./stream'); | ||
var es = require('event-stream'); | ||
var _ = require('lodash'); | ||
// var debug = require('debug')('mongodb-schema:wrapper'); | ||
/** | ||
* Convenience shortcut for parsing schemas. | ||
* @param {String} ns The namespace of the collection being parsed. | ||
* @param {Cursor|Array} docs An array of documents or a Cursor returned by `.find()` | ||
* @param {Function} fn Callback which will be passed `(err, schema)` | ||
* @return {Schema} | ||
* Convenience shortcut for parsing schemas. Accepts an array, stream or | ||
* MongoDB cursor object to parse documents from. | ||
* | ||
* @param {Cursor| | ||
* Array| | ||
* Stream} docs An array, a cursor, or a stream | ||
* @param {Function} fn Callback which will be passed `(err, schema)` | ||
*/ | ||
module.exports = function(ns, docs, fn) { | ||
var schema = new Schema({ | ||
ns: ns | ||
}); | ||
module.exports = function(docs, fn) { | ||
var src; | ||
// MongoDB Cursors | ||
@@ -24,17 +25,23 @@ if (docs.stream && typeof docs.stream === 'function') { | ||
// Arrays | ||
} else if (_.isArray(docs)) { | ||
src = es.readArray(docs); | ||
} else { | ||
src = es.readArray(docs); | ||
fn(new Error('Unknown input type for `docs`. Must be an array, ' | ||
+ 'stream or MongoDB Cursor.')); | ||
return; | ||
} | ||
// always use native parser (true) now. flag will eventually be removed | ||
// in an upcoming version. | ||
src.pipe(schema.stream(true)).pipe(es.wait(function() { | ||
fn.call(null, null, schema); | ||
})); | ||
return schema; | ||
var result; | ||
src.pipe(stream()) | ||
.on('data', function(data) { | ||
result = data; | ||
}) | ||
.on('error', function(err) { | ||
fn(err); | ||
}) | ||
.on('end', function() { | ||
fn(null, result); | ||
}); | ||
}; | ||
module.exports.Schema = Schema; | ||
module.exports.FieldCollection = require('./field-collection'); | ||
module.exports.TypeCollection = require('./type-collection'); | ||
module.exports.ValueCollection = require('./value-collection'); | ||
module.exports.stream = stream; |
{ | ||
"name": "mongodb-schema", | ||
"description": "Infer the probabilistic schema for a MongoDB collection.", | ||
"version": "4.2.4", | ||
"version": "5.0.0", | ||
"author": "Thomas Rueckstiess <thomas@rueckstiess.net>", | ||
@@ -16,3 +16,3 @@ "license": "Apache-2.0", | ||
"bin": { | ||
"mongodb-schema": "bin/mongodb-schema.js" | ||
"mongodb-schema": "bin/mongodb-schema" | ||
}, | ||
@@ -39,3 +39,3 @@ "scripts": { | ||
"entries": [ | ||
"bin/mongodb-schema.js", | ||
"bin/mongodb-schema", | ||
"index.js" | ||
@@ -45,5 +45,2 @@ ] | ||
"dependencies": { | ||
"ampersand-collection": "^1.4.5", | ||
"ampersand-collection-lodash-mixin": "^2.0.1", | ||
"ampersand-state": "4.8.2", | ||
"async": "^1.5.2", | ||
@@ -50,0 +47,0 @@ "event-stream": "^3.3.0", |
116
README.md
@@ -1,41 +0,80 @@ | ||
# mongodb-schema [![][npm_img]][npm_url] [![][travis_img]][travis_url] [![][coverage_img]][coverage_url] [![][gitter_img]][gitter_url] | ||
# mongodb-schema [![][npm_img]][npm_url] [![][travis_img]][travis_url] [![][coverage_img]][coverage_url] | ||
Infer a probabilistic schema for a MongoDB collection. | ||
A high-level view of the class interactions is as follows: | ||
## Usage | ||
 | ||
`mongodb-schema` can be used as a command line tool or programmatically in your application as a node module. | ||
## Example | ||
### Command line | ||
`mongodb-schema` doesn't do anything directly with `mongodb` so to try the examples we'll install the node.js driver. | ||
As well, we'll need some data in a collection to derive the schema of. | ||
To install mongodb-schema for command line use, run `npm install -g mongodb-schema`. This will add a new | ||
shell script which you can run directly from the command line. | ||
Make sure you have a `mongod` running on localhost on port 27017 (or change the example accordingly). Then, do: | ||
The command line tool expects a MongoDB connection URI and a namespace in the form `<database>.<collection>`. | ||
Without further arguments, it will sample 100 random documents from the collection and print a schema of | ||
the collection in JSON format to stdout. | ||
1. `npm install mongodb mongodb-schema` | ||
2. `mongo --eval "db.test.insert([{_id: 1, a: true}, {_id: 2, a: 'true'}, {_id: 3, a: 1}, {_id: 4}])" localhost:27017/test` | ||
``` | ||
mongodb-schema mongodb://localhost:27017 mongodb.fanclub | ||
``` | ||
Additional arguments change the number of samples (`--sample`), print additional statistics about the | ||
schema analysis (`--stats`), switch to a different output format (`--format`), or let you suppress the | ||
schema output altogether (`--no-output`) if you are only interested in the schema statistics. | ||
For more information, run | ||
``` | ||
mongodb-schema --help | ||
``` | ||
### API | ||
The following example demonstrates how `mongodb-schema` can be used programmatically from | ||
your node application. You need to additionally install the MongoDB node driver to follow | ||
along with this example. | ||
Make sure you have a `mongod` running on localhost on port 27017 (or change the example | ||
below accordingly). | ||
1. From your application folder, install the driver and `mongodb-schema` locally: | ||
``` | ||
npm install mongodb mongodb-schema | ||
``` | ||
2. (optional) If you don't have any data in your MongoDB instance yet, you can create a | ||
`test.data` collection with this command: | ||
``` | ||
mongo --eval "db.data.insert([{_id: 1, a: true}, {_id: 2, a: 'true'}, {_id: 3, a: 1}, {_id: 4}])" localhost:27017/test` | ||
``` | ||
3. Create a new file `parse-schema.js` and paste in the following code: | ||
```javascript | ||
var parseSchema = require('mongodb-schema'); | ||
var connect = require('mongodb'); | ||
connect('mongodb://localhost:27017/test', function(err, db){ | ||
if(err) return console.error(err); | ||
```javascript | ||
var parseSchema = require('mongodb-schema'); | ||
var connect = require('mongodb'); | ||
parseSchema('test.test', db.collection('test').find(), function(err, schema){ | ||
if(err) return console.error(err); | ||
connect('mongodb://localhost:27017/test', function(err, db) { | ||
if (err) return console.error(err); | ||
console.log(JSON.stringify(schema, null, 2)); | ||
db.close(); | ||
// here we are passing in a cursor as the first argument. You can | ||
// also pass in a stream or an array of documents directly. | ||
parseSchema(db.collection('data').find(), function(err, schema) { | ||
if (err) return console.error(err); | ||
console.log(JSON.stringify(schema, null, 2)); | ||
db.close(); | ||
}); | ||
}); | ||
}); | ||
``` | ||
4. When we run the above with `node parse-schema.js`, we'll see something | ||
like the following (some fields not present here for clarity): | ||
``` | ||
4. When we run the above with `node ./parse-schema.js`, we'll see output | ||
similar to this (some fields not present here for clarity): | ||
```javascript | ||
{ | ||
"count": 4, // parsed 4 documents | ||
"ns": "test.test", // namespace | ||
"fields": [ // an array of Field objects, @see `./lib/field.js` | ||
@@ -47,3 +86,2 @@ { | ||
"probability": 1, // all documents had an _id field | ||
"unique": 4, // 4 unique values found | ||
"has_duplicates": false, // therefore no duplicates | ||
@@ -75,3 +113,2 @@ "types": [ // an array of Type objects, @see `./lib/types/` | ||
], | ||
"unique": 3, | ||
"has_duplicates": false, // there were no duplicate values | ||
@@ -118,4 +155,9 @@ "types": [ | ||
### More Examples | ||
A high-level view of the schema tree structure is as follows: | ||
 | ||
## BSON Types | ||
`mongodb-schema` supports all [BSON types][bson-types]. | ||
@@ -129,9 +171,9 @@ Checkout [the tests][tests] for more usage examples. | ||
### Schema Depth | ||
#### Schema Depth | ||
The schema depth is defined as the maximum number of nested levels of keys in the schema. It does not matter if the subdocuments are nested directly or as elements of an array. An empty document has a depth of 0, whereas a document with some top-level keys but no nested subdocuments has a depth of 1. | ||
### Schema Width | ||
#### Schema Width | ||
The schema width is defined as the number of individual keys, added up over all nesting levels of the schema. Array values do not count towards the schema width. | ||
### Examples | ||
#### Examples | ||
@@ -227,9 +269,2 @@ ```js | ||
## Installation | ||
``` | ||
npm install --save mongodb-schema | ||
``` | ||
## Testing | ||
@@ -241,11 +276,2 @@ | ||
## Dependencies | ||
Under the hood, `mongodb-schema` uses [ampersand-state][ampersand-state] and | ||
[ampersand-collection][ampersand-collection] for modeling [Schema][schema], [Field][field]'s, and [Type][type]'s. | ||
**Note:** Currently we are pinning [ampersand-state][ampersand-state] to version 4.8.2 due | ||
to a backwards-breaking change introduced in version 4.9.x. For more details, see [ampersand-state issue #226](https://github.com/AmpersandJS/ampersand-state/issues/226). | ||
## License | ||
@@ -252,0 +278,0 @@ |
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 1 instance in 1 package
14
-17.65%293
9.74%0
-100%35386
-38.32%9
-57.14%317
-77.19%1
Infinity%- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed