Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

autopsy

Package Overview
Dependencies
Maintainers
3
Versions
22
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

autopsy - npm Package Compare versions

Comparing version 0.0.4 to 0.1.0

banner.txt
lib/connect.js
lib/mdb.js

72

bin/mdb.js
#!/usr/bin/env node
var os = require('os')
var path = require('path')
var which = require('which')
var through = require('through2')
var split = require('split')
var spawn = require('child_process').spawn
prepare(process.argv.slice(2), function (err, args) {
if (err) return console.error(err)
var mdb = spawn('expect', [path.join(__dirname, '../scripts/mdb')].concat(args))
var lastInput
process.stdin
.pipe(through(function (data, enc, cb) {
lastInput = data
if (data.length === 1 && data[0] === 10) {
process.stdout.write('> ')
}
cb(null, data)
}))
.pipe(mdb.stdin)
mdb.stdout
.pipe(split())
.pipe(through(function (data, enc, cb) {
if (compare(data, lastInput)) return cb()
if (!ready(data)) return cb()
cb(null, data)
process.stdout.write('\n> ')
}))
.pipe(process.stdout)
function ready (data) {
if (ready.rx.test(data + '')) {
ready.now = true
mdb.stdin.write('::load /mdb/mdb_v8.so\n')
}
return ready.now
}
ready.rx = /\[root@.+ ~\]# zlogin 7f3ba160-047c-4557-9e87-8157db23f205 mdb/
})
function compare (a, b) {
return (a + '').replace(/\n|\r/g, '').replace(/ /, '') === (b + '').replace(/\n|\r/g, '').replace(/ /, '')
}
function prepare (args, cb) {
if (args.length < 2) {
if (os.platform() !== 'linux') {
throw Error('On OSX we need both the linux core file and the exact node binary that generated it (e.g. in the linux environment)')
}
// for convenience on linux, we'll assume the current
// installed node binary
args.unshift(which.sync('node'))
}
spawn('expect', [path.join(__dirname, '../scripts/clean')])
.on('close', function () {
spawn('expect', [path.join(__dirname, '../scripts/cp')].concat(args))
.on('close', function () {
cb(null, args.map(function (f) { return path.basename(f) }))
})
.on('error', cb)
})
.on('error', cb)
}
require('../lib/mdb')(process.argv.slice(2))

41

bin/setup.js

@@ -36,9 +36,2 @@ #!/usr/bin/env node

if (err) return cb(err)
// have to copy the file so we still have it,
// otherwise the naughty virtual box steals it
fs.writeFileSync(
path.join(__dirname, '../assets/smartos-disk1.vmdk'),
fs.readFileSync(path.join(__dirname, '../assets/_smartos-disk1.vmdk')))
var args = [

@@ -49,32 +42,20 @@ 'smartos',

'--device', '0',
'--type', 'hdd',
'--medium', path.join(__dirname, '../assets/smartos-disk1.vmdk')
'--type', 'dvddrive',
'--medium', path.join(__dirname, '../assets/smartos.iso')
]
vbm.command.exec('storageattach', args, function (err, code, output) {
if (err) return cb(err)
var args = [
'smartos',
'--storagectl', 'IDE Controller',
'--port', '1',
'--device', '0',
'--type', 'dvddrive',
'--medium', path.join(__dirname, '../assets/smartos-latest.iso')
]
// var args = [
// 'smartos',
// '--macaddress1', '080027B0011A'
// ]
vbm.command.exec('storageattach', args, function (err, code, output) {
if (err) return cb(err)
var args = [
'smartos',
'--macaddress1', '080027B0011A'
]
vbm.command.exec('modifyvm', args, function (err, code, output) {
if (err) return cb(err)
cb()
})
})
// vbm.command.exec('modifyvm', args, function (err, code, output) {
// if (err) return cb(err)
cb()
// })
})
})
}

6

bin/start.js

@@ -5,5 +5,8 @@ #!/usr/bin/env node

function up (err) {
function up (err, firstBoot) {
if (err) return console.error(err)
console.log('vm up, waiting for ready state')
if (firstBoot) {
console.log('No snapshot, first boot will take a while (up to 5 mins)')
}
}

@@ -14,2 +17,3 @@

console.log('vm ready')
process.exit(0)
}

3

lib/assets-uri.js

@@ -1,1 +0,2 @@

module.exports = 'https://s3-us-west-2.amazonaws.com/autopsy-assets/assets-191115.tar.gz'
module.exports =
'https://s3-us-west-2.amazonaws.com/autopsy-assets/assets-221115.tar.gz'

17

lib/download.js

@@ -8,5 +8,7 @@ var path = require('path')

var eos = require('end-of-stream')
var debug = require('debug')('autopsy:download')
var pkg = require('../package.json')
var extract = tar.extract()
var assets = path.join(__dirname, '..', 'assets')
module.exports = download

@@ -19,6 +21,5 @@

var partFile = path.join(__dirname, '..', 'assets.' + pkg.version + '.download')
debug('tarball saving to ' + partFile)
var assets = [
'assets/_smartos-disk1.vmdk',
'assets/smartos-latest.iso',
'assets/smartos.iso',
'assets/smartos.ovf'

@@ -34,2 +35,3 @@ ]

req.on('start', function (res) {
debug('starting download')
var pro = progress({

@@ -59,3 +61,5 @@ time: 100,

fs.mkdirSync(path.join(__dirname, '..', 'assets'))
if (!fs.existsSync(path.join(__dirname, '..', 'assets'))) {
fs.mkdirSync(path.join(__dirname, '..', 'assets'))
}

@@ -67,2 +71,3 @@ extract.on('entry', function (header, stream, next) {

debug('outputing ' + header.name + ' to ' + path.join(__dirname, '..', header.name))
stream

@@ -74,2 +79,3 @@ .resume()

debug('decompressing and extracting ' + partFile)
eos(

@@ -87,3 +93,4 @@ fs.createReadStream(partFile)

function clean () {
if (!fs.existsSync(assets)) return
if (!fs.existsSync(assets)) { return }
debug('assets folder already exists, attempting to remove')
fs.readdirSync(assets)

@@ -90,0 +97,0 @@ .map(function (f) { return path.join(assets, f) })

94

lib/start.js
var path = require('path')
var eos = require('end-of-stream')
var vbm = require('vboxmanage')
var spawn = require('child_process').spawn
var connect = require('./connect')
var debug = require('debug')('autopsy:start')
var SNAPSHOT_NAME = 'init'
var TIME_TILL_GRUB = 3500
var SSH_RETRY_WAIT = 10000
module.exports = function (cb, ready) {
snapshotExists(SNAPSHOT_NAME, function (err, exists) {
if (err) return cb && cb(err)
if (exists) {
debug('restoring snapshot for fast vm load')
return loadSnapshot(function (err) {
if (err) { return cb(err) }
start(cb, ready)
})
}
debug('no snapshot, entering long boot')
start(cb, ready, true)
})
}
function start (cb, ready, createInitialSnapshot) {
debug('attempting to start smartos vm')
vbm.instance.start('smartos', function (err) {
if (err) return cb && cb(err)
debug('waiting ' + TIME_TILL_GRUB + ' for GRUB loader screen')
setTimeout(function () {
// send the return key so we don't wait for grub
debug('attempting to send <Enter> key press into vm to bypass grub menu')
vbm.command.exec('controlvm', 'smartos keyboardputscancode 1c 9c'.split(' '), function () {
cb && cb()
function check () {
spawn('expect', [
path.join(__dirname, '../scripts/check')
]).on('close', function (code) {
if (!code) return ready()
if (code === 255) {
return setTimeout(check, 1000)
}
ready(Error('scripts/check got code ' + code))
cb && cb(null, createInitialSnapshot)
function check() {
check.count -= 1
if (check.count < 1) return
debug('running a check against ssh connection, remaining attempts: ' + check.count)
var ssh = connect.ssh()
eos(ssh, function () {
debug('failed to connect to ssh retrying in ' + SSH_RETRY_WAIT + 'ms')
setTimeout(function () {
check()
}, SSH_RETRY_WAIT)
})
ssh
.on('ready', function () {
if (!createInitialSnapshot) { return ready() }
debug('first run of vm, creating initialisation snapshot')
takeSnapshot(ready)
})
.on('error', ready)
}
if (!(ready instanceof Function)) return
check.count = 30
if (!(ready instanceof Function)) { return }
check()
})
}, 3500)
}, TIME_TILL_GRUB)
})
}
function snapshotExists(name, cb) {
debug('checking if snapshot exists')
var args = ['smartos', 'list', '--machinereadable']
vbm.command.exec('snapshot', args, function (err, code, list) {
if (err) { return cb(err) }
var exists = list.split('\n').some(function (s) {
return s === 'SnapshotName="' + name + '"'
})
debug('snapshot does' + (exists ? '' : ' not') + ' exist')
cb(null, exists)
})
}
function takeSnapshot(cb) {
debug('attempting to take a snapshot')
var args = ['smartos', 'take', 'init']
vbm.command.exec('snapshot', args, function (err) {
debug(err ? 'unable to take snapshot' : 'snapshot successfully taken')
cb(err)
})
}
function loadSnapshot(cb) {
debug('attempting to load snapshot')
var args = ['smartos', 'restore', SNAPSHOT_NAME]
vbm.command.exec('snapshot', args, function (err) {
debug(err ? 'unable to load snapshot' : 'snapshot successfully loaded')
cb(err)
})
}

15

package.json
{
"name": "autopsy",
"version": "0.0.4",
"version": "0.1.0",
"description": "dissect dead node service core dumps with mdb via a smart os vm",
"main": "index.js",
"preferGlobal": true,
"files": [
"banner.txt",
"bin",
"lib"
],
"bin": {

@@ -19,4 +24,3 @@ "autopsy": "./bin/mdb.js",

"start": "node ./bin/start.js",
"stop": "node ./bin/stop.js",
"postinstall": "npm run setup"
"stop": "node ./bin/stop.js"
},

@@ -26,6 +30,11 @@ "author": "David Mark Clements",

"dependencies": {
"debug": "^2.2.0",
"end-of-stream": "^1.1.0",
"fast-download": "^0.3.5",
"keypress": "^0.2.1",
"progress-stream": "^1.1.1",
"scp2": "^0.2.2",
"split": "^1.0.0",
"ssh-connect-prompt": "0.0.2",
"ssh2": "^0.4.12",
"tar-stream": "^1.3.1",

@@ -32,0 +41,0 @@ "through2": "^2.0.0",

240

Readme.md

@@ -5,75 +5,59 @@ # Autopsy

## Why?
## About
mdb is an awesome debugger that comes with smart os.
There is a Unix-like operating system called SmartOS whose
ancestry represents a strong investment in low-level
introspection tools (such as dtrace for instance).
There's an extension for it that allows you to
postmortem node core files
Once such tool is `mdb`, a high quality modular debugger which
ships with SmartOS - it can be used to inspect execution
context from the kernel to application layers.
However, a tremendous amount of people run node
on linux (not least because of docker and what not).
Some rather clever people wrote a debugging module called `mdb_v8`
that allows introspection of node core dumps from a high level (e.g.
inspecting closure scope) to a low level (e.g. memory addresses).
But it turns out that mdb can analyse linux core files,
we just have to give it the core file and the node binary
that was running when the core file was generated.
It turns out that `mdb` can analyse Linux core files as well
as SmartOS core files. We just have provide the core file
and the node binary that was running when the core dump was
generated.
The problem is, actually getting you linux core file
into an environment that is running a version of mdb
that this can work with is... painful.
`autopsy` installs a a SmartOS VM and then acts as a
stdio proxy to `mdb`.
So, autopsy abstracts that pain away, and installs an `autopsy`
executable on linux that essentially acts as a proxy to the
mdb client within the smartos vm.
For using mdb see the [mdb reference docs][]
You can also run autopsy on OS X, but you'll need the linux
node binary and core file to pass to it.
<!--
TODO: gif screen cast
-->
## Prerequisites
## Setup
* VirtualBox
* 2gb of RAM for VM
* The VM runs entirely in RAM and mdb can be memory intensive also, 2gb is a safe bet
### Preqrequisites
## Install
Autopsy depends on virtual box, on ubuntu/debian we can do
Install autopsy from npm:
```sh
sudo apt-get install virtualbox
```
Sometimes the virtualbox package for ubuntu seems to have
problems with kernel versions (particularly if we're installing
virtualbox into an ubuntu vm). We can check whether the virtualbox
install was successful with:
```sh
VBoxManage --version
sudo npm install -g autopsy
```
If we see some error output about the character device /dev/vboxdrv
does not exist then there's a problem.
Once finished the following executables will be available
* autopsy-setup - runs setup
* autopsy-start - starts the vm
* autopsy-stop - stops the vm
* autopsy-status - gets vm status
* autopsy - provides the CLI proxy to mdb in the vm
In this case we can grab the latest `.deb` file from the virtualbox site, e.g.:
Next, set up the VM
```sh
$ sudo apt-get purge virtualbox
$ curl http://download.virtualbox.org/virtualbox/5.0.10/virtualbox-5.0_5.0.10-104061~Ubuntu~trusty_amd64.deb > vbox.deb
$ sudo apt-get install linux-headers-`uname -r`
$ sudo dpkg -i vbox.deb
```
### VM Setup
To find the right deb package for your Linux distro see <https://www.virtualbox.org/wiki/Linux_Downloads>.
There's also currently a hard dependency on `expect`
```sh
sudo apt-get install expect
```
On OS X - well we can work it out ;)
### Install
autopsy-setup
```
sudo npm install -g autopsy
```

@@ -83,30 +67,29 @@ This will install autopsy on the system, download smartos

The VM assets download is ~450mb, in testing on a fairly decent
connection, setup from start to finish (not including npm dep installs)
takes around 1.5 minutes. This is because we're using multithreaded
downloading and host the assets on S3.
Assets for the VM are ~150mb and downloads from S3.
Once finished the following executables will be available
If setup is interupted for any reason (including network failure
during assets download), simply try again. Partial downloads will
be resumed.
* autopsy-start - starts the vm
* autopsy-stop - stops the vm
* autopsy-status - gets vm status
* autopsy-setup - runs setup (only needful for troubleshooting)
* autopsy - provides the CLI proxy to mdb in the vm
## Starting the VM
### Resuming Setup
Before we can do an autopsy the VM needs to be running.
If postinstall setup is interupted for any reason (including network failure
during assets download), try again with
Simply run
```sh
autopsy-setup
```
autopsy-start
```
If there was a partial download, it should resume rather than restart.
Autopsy takes a snapshot of the initial VM state on first run to
optimize subsequent boots, so the first `autopsy-start` will be
the longest.
## autopsy
The VM runs SmartOS completely in ram (there are no zones).
This means VM state is immutable.
The autopsy command takes the following args
## Performing an autopsy
The `autopsy` command takes the following args
```sh

@@ -116,3 +99,3 @@ autopsy [node-binary] core-file

On OS X the node binary is not optional, on linux
On OS X the node binary is not optional, on Linux
if not supplied the current installed node binary

@@ -131,35 +114,10 @@ will be used.

## Stopping the VM
## Generating a core file
When we're done we may wish to free memory by stopping the VM with
In production, if we run our node processes with `--abort-on-uncaught-exception` we will always get a core dump when a process crashes (that is,
as long as our linux environment is set up correctly)
You can also manually generate a core file using `process.abort()`.
Finally a core file can also be obtained by attaching `gdb` to a running processing and executing `generate-core`.
## Setting up Linux to generate core files
If you're using an ubuntu server (and probably debian etc. etc.) you may have apport installed - this intercepts core files so we need to get rid of it
```
sudo apt-get purge apport
autopsy-stop
```
Next you need to make sure that linux is configured to allocate
space for the core file, like so
```
ulimit -c unlimited
```
Put this in a start up script and what not.
## Why's it so big?
We'll get it smaller (hopefully), this is a first pass
and we're focusing on functionality. But the size it's
because we're like.. running an entire virtual machine.
## Example

@@ -191,48 +149,38 @@

## todo
## How to generate a core file
* get up arrow (for history) working
In production, if we run our node processes with `--abort-on-uncaught-exception` we will always get a core dump when a process crashes (that is,
as long as our linux environment is set up correctly)
## Future
You can also manually generate a core file using `process.abort()`.
* reduce size of everything
* maybe work with saved machine state to decrease boot time
* can we get rid of the dep on virtual box I wonder?
* work out if this is a problem "mdb: warning: librtld_db failed to initialize; shared library information will not be available" - may need a patch in the vm
* get ssh keys into the vm instead of passing a pw with expect, granted the pw being plain text in this case doesn't matter, but we might be able to get rid of the dependency on expect if we get rid of the pw.
* vm shouldn't need 5gb of ram, sort that out.
* extend this into another project that runs node processes in smart os zones (just like docker containers) - but with the added benefit of native smart os core dumps which can be analysed with mdb
Finally a core file can also be obtained by attaching `gdb` to a running processing and executing `generate-core`.
## Setting up Linux to generate core files
If you're using an ubuntu server (and probably debian etc. etc.) you may have apport installed - this intercepts core files so we need to get rid of it
## mdb_v8 upgrade notes
```
sudo apt-get purge apport
```
The latest smartos comes with an old version of mdb_v8, as of autopsy 0.0.2 the vm runs mdb_v8 1.2.2 (latest at time of writing)
To upgrade the v8 version (without waiting for an autopsy release) we can perform the following steps
Next you need to make sure that linux is configured to allocate
space for the core file, like so
1. login to the vm `ssh -p 2222 root@localhost` pw: mdb
2. login to the zone `zlogin 7f3ba160-047c-4557-9e87-8157db23f205`
3. `mkdir /mdb && cd /mdb`
4. `pkgin install gcc49-4.9.1 gmake-4.0 git`
5. `git clone https://github.com/joyent/mdb_v8`
6. `cd mdb_v8 && make`
7. `cp mdb_v8/builds/amd64/mdb_v8.so .`
```
ulimit -c unlimited
```
At this point we have successfully upgraded to latest mdb_v8, however
we have a lot of extra dev packages installed in the vm making it much
less lean. So, we may want to copy the `mdb_v8.so` file from the vm, like so:
Put this in a start up script and what not.
```sh
scp -P 2222 root@localhost:/zones/7f3ba160-047c-4557-9e87-8157db23f205/root/mdb/mdb_v8.so .
```
## Caveats
Then recreate the vm (follow removing the vm below, then `npm run setup`) and copy the file back in (this is what we do for releases).
### Port 2222
```sh
scp -P 2222 mdb_v8.so root@localhost:/zones/7f3ba160-047c-4557-9e87-8157db23f205/root/mdb
```
The VM currently maps port 2222 to the port 22 (ssh), at the moment
is non-configurable - so to use autopsy port 2222 needs to be free on the
host system.
### Removing the VM
## removing the vm
Currently there's no command for removing the vm, follow these steps, in order

@@ -244,12 +192,28 @@

### Singleton VM - Run Globally
## Caveats
We recommend installing globally, since there can (currently) only be one
smartos vm.
* We recommend installing globally, since there can only be one
smartos vm.
* If the assets folder or any parent folder is moved or
renamed, the vm will fail to start (because it won't be able
to locate the the iso and vmdk files). In this case you would need to
### Virtualbox Filesystem Coupling
If the smartos.iso file or any parent folder is moved/renamed
the vm will fail to start because virtualbox won't be able
to locate the the iso. In this case you would need to
manually update virtual box with the paths.
[mdb reference docs]: https://github.com/joyent/mdb_v8/blob/master/docs/usage.md#node-specific-mdb-command-reference
## Debug Logs
For troubleshooting (or the curious), debugging can be turned
on like so
```
DEBUG=autopsy:* <cmd>
```
At present the following commands have debug output
* autopsy
* autopsy-setup
* autopsy-start
[mdb reference docs]: https://github.com/joyent/mdb_v8/blob/master/docs/usage.md#node-specific-mdb-command-reference
.npmignore
scripts/check
scripts/clean
scripts/cp
scripts/mdb
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc