rets-client - npm Package Compare versions

Comparing version 2.0.7 to 3.0.0

lib/api.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+Promise = require('bluebird')
+
+Client = require('./client')
+replyCodes = require('./utils/replyCodes')
+errors = require('./utils/errors')
+
+
+###
+  Available login settings:
+      loginUrl: RETS login URL (i.e http://<MLS_DOMAIN>/rets/login.ashx)
+      username: username credential
+      password: password credential
+      version: rets version
+
+      //RETS-UA-Authorization
+      userAgent
+      userAgentPassword
+      sessionId
+###
+
+module.exports =
+  RetsReplyError: errors.RetsReplyError
+  RetsServerError: errors.RetsServerError
+  Client: Client
+  getAutoLogoutClient: Client.getAutoLogoutClient
+  getReplyTag: replyCodes.getReplyTag

lib/clientModules/metadata.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+logger = require('winston')
+Promise = require('bluebird')
+through2 = require('through2')
+
+replyCodes = require('../utils/replyCodes')
+retsHttp = require('../utils/retsHttp')
+retsParsing = require('../utils/retsParsing')
+
+
+_getMetadataImpl = (retsSession, type, options) -> new Promise (resolve, reject) ->
+  context = retsParsing.getStreamParser(type)
+  retsHttp.streamRetsMethod('getMetadata', retsSession, options, context.fail)
+  .pipe(context.parser)
+
+  result =
+    results: []
+    type: type
+  currEntry = null
+
+  context.retsStream.pipe through2.obj (event, encoding, callback) ->
+    switch event.type
+      when 'data'
+        currEntry.metadata.push(event.payload)
+      when 'metadataStart'
+        currEntry =
+          info: event.payload
+          metadata: []
+        result.results.push(currEntry)
+      when 'metadataEnd'
+        currEntry.info.rowsReceived = event.payload
+      when 'status'
+        for own key, value of event.payload
+          result[key] = value
+      when 'done'
+        for own key, value of event.payload
+          result[key] = value
+        resolve(result)
+      when 'error'
+        reject(event.payload)
+    callback()
+
+
+###
+# Retrieves RETS Metadata.
+#
+# @param type Metadata type (i.e METADATA-RESOURCE, METADATA-CLASS)
+# @param id Metadata id
+# @param format Data format (i.e. COMPACT, COMPACT-DECODED), defaults to 'COMPACT'
+###
+
+getMetadata = (type, id, format='COMPACT') -> Promise.try () =>
+  if !type
+    throw new Error('Metadata type is required')
+  if !id
+    throw new Error('Resource type id is required (or for some types of metadata, "0" retrieves for all resource types)')
+  options =
+    Type: type
+    Id: id
+    Format: format
+  retsHttp.callRetsMethod('getMetadata', @retsSession, options)
+  .then (result) ->
+    result.body
+
+
+###
+# Helper that retrieves RETS system metadata
+###
+
+getSystem = () ->
+  @getMetadata('METADATA-SYSTEM')
+  .then (rawXml) -> new Promise (resolve, reject) ->
+    result = {}
+    retsParser = retsParsing.getSimpleParser(reject)
+
+    gotMetaDataInfo = false
+    gotSystemInfo = false
+    retsParser.parser.on 'startElement', (name, attrs) ->
+      switch name
+        when 'METADATA-SYSTEM'
+          gotMetaDataInfo = true
+          result.metadataVersion = attrs.Version
+          result.metadataDate = attrs.Date
+        when 'SYSTEM'
+          gotSystemInfo = true
+          result.systemId = attrs.SystemID
+          result.systemDescription = attrs.SystemDescription
+
+    retsParser.parser.on 'endElement', (name) ->
+      if name != 'RETS'
+        return
+      retsParser.finish()
+      if !gotSystemInfo || !gotMetaDataInfo
+        reject(new Error('Failed to parse data'))
+      else
+        resolve(result)
+    
+    retsParser.parser.write(rawXml)
+    retsParser.parser.end()
+
+
+module.exports = (_retsSession) ->
+  if !_retsSession
+    throw new Error('System data not set; invoke login().')
+  
+
+  _getParsedMetadataFactory = (type, format='COMPACT') ->
+    (id, classType) -> Promise.try () ->
+      if !id
+        throw new Error('Resource type id is required (or for some types of metadata, "0" retrieves for all resource types)')
+      options =
+        Type: type
+        Id: if classType then "#{id}:#{classType}" else id
+        Format: format
+      _getMetadataImpl(_retsSession, type, options)
+  
+  
+  _getParsedAllMetadataFactory = (type, format='COMPACT') ->
+    options =
+      Type: type
+      Id: '0'
+      Format: format
+    () -> _getMetadataImpl(_retsSession, type, options)
+  
+  
+  retsSession: Promise.promisify(_retsSession)
+  getMetadata: getMetadata
+  getSystem: getSystem
+  getResources:       _getParsedMetadataFactory('METADATA-RESOURCE')
+  getForeignKeys:     _getParsedMetadataFactory('METADATA-FOREIGNKEYS')
+  getClass:           _getParsedMetadataFactory('METADATA-CLASS')
+  getTable:           _getParsedMetadataFactory('METADATA-TABLE')
+  getLookups:         _getParsedMetadataFactory('METADATA-LOOKUP')
+  getLookupTypes:     _getParsedMetadataFactory('METADATA-LOOKUP_TYPE')
+  getObject:          _getParsedMetadataFactory('METADATA-OBJECT')
+  getAllForeignKeys:  _getParsedAllMetadataFactory('METADATA-FOREIGNKEYS')
+  getAllClass:        _getParsedAllMetadataFactory('METADATA-CLASS')
+  getAllTable:        _getParsedAllMetadataFactory('METADATA-TABLE')
+  getAllLookups:      _getParsedAllMetadataFactory('METADATA-LOOKUP')
+  getAllLookupTypes:  _getParsedAllMetadataFactory('METADATA-LOOKUP_TYPE')

lib/clientModules/object.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+logger = require('winston')
+streamBuffers = require('stream-buffers')
+Promise = require('bluebird')
+
+multipart = require('../utils/multipart')
+
+
+###
+# Retrieves RETS object data.
+#
+# @param resourceType Rets resource type (ex: Property)
+# @param objectType Rets object type (ex: LargePhoto)
+# @param objectId Object identifier
+###
+
+getObject = (resourceType, objectType, objectId) ->
+  if !resourceType
+    throw new Error('Resource type id is required')
+  if !objectType
+    throw new Error('Object type id is required')
+  if !objectId
+    throw new Error('Object id is required')
+  options =
+    Type: objectType
+    Id: objectId
+    Resource: resourceType
+  
+  # prepare stream buffer for object data
+  writableStreamBuffer = new (streamBuffers.WritableStreamBuffer)(
+    initialSize: 100 * 1024
+    incrementAmount: 10 * 1024)
+  req = @retsSession(options)
+  
+  #pipe object data to stream buffer
+  new Promise (resolve, reject) ->
+    req.pipe(writableStreamBuffer)
+    req.on('error', reject)
+    contentType = null
+    req.on 'response', (_response) ->
+      contentType = _response.headers['content-type']
+    req.on 'end', ->
+      resolve
+        contentType: contentType
+        data: writableStreamBuffer.getContents()
+
+###
+# Helper that retrieves a list of photo objects.
+#
+# @param resourceType Rets resource type (ex: Property)
+# @param photoType Photo object type, based on getObjects meta call (ex: LargePhoto, Photo)
+# @param matrixId Photo matrix identifier.
+#
+# Each item in resolved data list is an object with the following data elements:
+#   buffer: <data buffer>,
+#   mime: <data buffer mime type>,
+#   description: <data description>,
+#   contentDescription: <data content description>,
+#   contentId: <content identifier>,
+#   objectId: <object identifier>
+###
+
+getPhotos = (resourceType, photoType, matrixId) ->
+  @getObject(resourceType, photoType, matrixId + ':*')
+  .then (result) ->
+    multipartBoundary = result.contentType.match(/boundary=(?:"([^"]+)"|([^;]+))/ig)[0].match(/[^boundary=^"]\w+[^"]/ig)[0]
+    if !multipartBoundary
+      throw new Error('Could not find multipart boundary')
+    multipart.parseMultipart(new Buffer(result.data), multipartBoundary)
+    .catch (err) ->
+      logger.error err
+      throw new Error('Error parsing multipart data')
+
+
+module.exports = (_retsSession) ->
+  if !_retsSession
+    throw new Error('System data not set; invoke login().')
+  retsSession: Promise.promisify(_retsSession)
+  getObject: getObject
+  getPhotos: getPhotos

lib/clientModules/object.stream.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+
+###
+  This file is a placeholder for the object streaming features to be added in v3.1.0
+###

lib/clientModules/search.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+logger = require('winston')
+Promise = require('bluebird')
+through2 = require('through2')
+
+queryOptionHelpers = require('../utils/queryOptions')
+errors = require('../utils/errors')
+hex2a = require('../utils/hex2a')
+replyCodes = require('../utils/replyCodes')
+retsParsing = require('../utils/retsParsing')
+retsHttp = require('../utils/retsHttp')
+
+
+###
+# Invokes RETS search operation.
+#
+# @param _queryOptions Search query options.
+#    See RETS specification for query options.
+#
+#    Default values for query params:
+#
+#       queryType:'DMQL2',
+#       format:'COMPACT-DECODED',
+#       count:1,
+#       standardNames:0,
+#       restrictedIndicator:'***',
+#       limit:"NONE"
+###
+
+searchRets = (queryOptions) -> Promise.try () =>
+  finalQueryOptions = queryOptionHelpers.normalizeOptions(queryOptions)
+  retsHttp.callRetsMethod('search', @retsSession, finalQueryOptions)
+  .then (result) ->
+    result.body
+
+
+###
+#
+# Helper that performs a targeted RETS query and parses results.
+#
+# @param searchType Rets resource type (ex: Property)
+# @param classType Rets class type (ex: RESI)
+# @param query Rets query string. See RETS specification - (ex: MatrixModifiedDT=2014-01-01T00:00:00.000+)
+# @param options Search query options (optional).
+#    See RETS specification for query options.
+#
+#    Default values for query params:
+#
+#       queryType:'DMQL2',
+#       format:'COMPACT-DECODED',
+#       count:1,
+#       standardNames:0,
+#       restrictedIndicator:'***',
+#       limit:"NONE"
+#
+#       Please note that queryType and format are immutable.
+###
+
+query = (resourceType, classType, queryString, options={}) -> new Promise (resolve, reject) =>
+  result =
+    results: []
+    maxRowsExceeded: false
+  currEntry = null
+
+  @stream.query(resourceType, classType, queryString, options)
+  .pipe through2.obj (event, encoding, callback) ->
+    switch event.type
+      when 'data'
+        result.results.push(event.payload)
+      when 'status'
+        for own key, value of event.payload
+          result[key] = value
+      when 'count'
+        result.count = event.payload
+      when 'done'
+        for own key, value of event.payload
+          result[key] = value
+        resolve(result)
+      when 'error'
+        reject(event.payload)
+    callback()
+
+
+module.exports = (_retsSession) ->
+  if !_retsSession
+    throw new Error('System data not set; invoke login().')
+  retsSession: Promise.promisify(_retsSession)
+  searchRets: searchRets
+  query: query
+  stream: require('./search.stream')(_retsSession)

lib/clientModules/search.stream.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+through2 = require('through2')
+
+queryOptionHelpers = require('../utils/queryOptions')
+retsHttp = require('../utils/retsHttp')
+retsParsing = require('../utils/retsParsing')
+
+
+###
+# Invokes RETS search operation and streams the resulting XML.
+#
+# @param _queryOptions Search query options.
+#    See RETS specification for query options.
+#
+#    Default values for query params:
+#
+#       queryType:'DMQL2',
+#       format:'COMPACT-DECODED',
+#       count:1,
+#       standardNames:0,
+#       restrictedIndicator:'***',
+#       limit:"NONE"
+###
+
+searchRets = (queryOptions) -> Promise.try () =>
+  finalQueryOptions = queryOptionHelpers.normalizeOptions(queryOptions)
+  resultStream = through2()
+  httpStream = retsHttp.streamRetsMethod 'search', @retsSession, finalQueryOptions, (err) ->
+    httpStream.unpipe(resultStream)
+    resultStream.emit('error', err)
+  httpStream.pipe(resultStream)
+
+
+###
+#
+# Helper that performs a targeted RETS query and streams parsed (or semi-parsed) results
+#
+# @param searchType Rets resource type (ex: Property)
+# @param classType Rets class type (ex: RESI)
+# @param query Rets query string. See RETS specification - (ex: MatrixModifiedDT=2014-01-01T00:00:00.000+)
+# @param options Search query options (optional).
+#    See RETS specification for query options.
+# @param rawData flag indicating whether to skip parsing of column and data elements.
+#
+#    Default values for query params:
+#
+#       queryType:'DMQL2',
+#       format:'COMPACT-DECODED',
+#       count:1,
+#       standardNames:0,
+#       restrictedIndicator:'***',
+#       limit:"NONE"
+#
+#       Please note that queryType and format are immutable.
+###
+
+query = (resourceType, classType, queryString, options={}, rawData=false) ->
+  baseOpts =
+    searchType: resourceType
+    class: classType
+    query: queryString
+  queryOptions = queryOptionHelpers.mergeOptions(baseOpts, options)
+
+  # make sure queryType and format will use the searchRets defaults
+  delete queryOptions.queryType
+  delete queryOptions.format
+  finalQueryOptions = queryOptionHelpers.normalizeOptions(queryOptions)
+  expectRows = "#{finalQueryOptions.count}" != "2"
+
+  context = retsParsing.getStreamParser(null, rawData, expectRows)
+  retsHttp.streamRetsMethod('search', @retsSession, finalQueryOptions, context.fail)
+  .pipe(context.parser)
+  
+  context.retsStream
+
+
+module.exports = (_retsSession) ->
+  if !_retsSession
+    throw new Error('System data not set; invoke login().')
+  retsSession: _retsSession
+  query: query
+  searchRets: searchRets

lib/utils/auth.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+logger = require('winston')
+Promise = require('bluebird')
+
+retsParsing = require('./retsParsing')
+retsHttp = require('./retsHttp')
+
+
+###
+# Executes RETS login routine.
+###
+
+login = (retsSession) ->
+  retsHttp.callRetsMethod('login', Promise.promisify(retsSession), {})
+  .then (retsResponse) -> new Promise (resolve, reject) ->
+    systemData =
+      retsVersion: retsResponse.response.headers['rets-version']
+      retsServer: retsResponse.response.headers.server
+    
+    retsParser = retsParsing.getSimpleParser(reject)
+    
+    gotData = false
+    retsParser.parser.on 'text', (text) ->
+      if retsParser.currElementName != 'RETS-RESPONSE'
+        return
+      gotData = true
+      keyVals = text.split('\r\n')
+      for keyVal in keyVals
+        split = keyVal.split('=')
+        if split.length > 1
+          systemData[split[0]] = split[1]
+
+    retsParser.parser.on 'endElement', (name) ->
+      if name != 'RETS'
+        return
+      retsParser.finish()
+      if !gotData
+        reject(new Error('Failed to parse data'))
+      else
+        resolve(systemData)
+
+    retsParser.parser.write(retsResponse.body)
+    retsParser.parser.end()
+
+
+###
+# Logouts RETS user
+###
+
+logout = (retsSession) ->
+  retsHttp.callRetsMethod('logout', Promise.promisify(retsSession), {})
+  .then (result) ->
+    logger.debug 'Logout success'
+
+
+module.exports =
+  login: login
+  logout: logout

lib/utils/errors.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+replyCodes = require('./replyCodes')
+
+
+class RetsReplyError extends Error
+  constructor: (@replyCode, @replyText) ->
+    @name = 'RetsReplyError'
+    @replyTag = if replyCodes.tagMap[@replyCode]? then replyCodes.tagMap[@replyCode] else 'unknown reply code'
+    @message = "RETS Server replied with an error code - ReplyCode #{@replyCode} (#{@replyTag}); ReplyText: #{@replyText}"
+    Error.captureStackTrace(this, RetsReplyError)
+
+
+class RetsServerError extends Error
+  constructor: (@retsMethod, @httpStatus, @httpStatusMessage) ->
+    @name = 'RetsServerError'
+    @message = "Error while attempting #{@retsMethod} - HTTP Status #{@httpStatus} returned (#{@httpStatusMessage})"
+    Error.captureStackTrace(this, RetsServerError)
+
+
+module.exports =
+  RetsReplyError: RetsReplyError
+  RetsServerError: RetsServerError

lib/utils/hex2a.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+
+hex2a = (hex) ->
+  if !hex
+    return null
+  
+  # force conversion
+  hex = hex.toString()
+  
+  str = ''
+  i = 0
+  while i < hex.length
+    str += String.fromCharCode(parseInt(hex.substr(i, 2), 16))
+    i += 2
+  str
+
+module.exports = hex2a

lib/utils/multipart.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+logger = require('winston')
+MultipartParser = require('formidable/lib/multipart_parser').MultipartParser
+Stream = require('stream').Stream
+StringDecoder = require('string_decoder').StringDecoder
+streamBuffers = require('stream-buffers')
+Promise = require('bluebird')
+
+
+# Multipart parser derived from formidable library. See https://github.com/felixge/node-formidable
+
+
+parseMultipart = (buffer, _multipartBoundary) -> Promise.try () ->
+  parser = getParser(_multipartBoundary)
+  if parser instanceof Error
+    return Promise.reject(parser)
+  parser.write(buffer)
+  dataBufferList = []
+  for streamBuffer in parser.streamBufferList
+    dataBufferList.push
+      buffer: streamBuffer.streamBuffer.getContents()
+      mime: streamBuffer.mime
+      description: streamBuffer.description
+      contentDescription: streamBuffer.contentDescription
+      contentId: streamBuffer.contentId
+      objectId: streamBuffer.objectId
+  dataBufferList
+
+
+getParser = (_multipartBoundary) ->
+  streamBufferList = []
+  parser = new MultipartParser()
+  headerField = ''
+  headerValue = ''
+  part = {}
+  encoding = 'utf8'
+  ended = false
+  maxFields = 1000
+  maxFieldsSize = 2 * 1024 * 1024
+
+  parser.onPartBegin = () ->
+    part = new Stream()
+    part.readable = true
+    part.headers = {}
+    part.name = null
+    part.filename = null
+    part.mime = null
+    part.transferEncoding = 'binary'
+    part.transferBuffer = ''
+    headerField = ''
+    headerValue = ''
+
+  parser.onHeaderField = (b, start, end) ->
+    headerField += b.toString(encoding, start, end)
+
+  parser.onHeaderValue = (b, start, end) ->
+    headerValue += b.toString(encoding, start, end)
+
+  parser.onHeaderEnd = () =>
+    headerField = headerField.toLowerCase()
+    part.headers[headerField] = headerValue
+    if headerField == 'content-disposition'
+      m = headerValue.match(/\bname="([^"]+)"/i)
+      if m
+        part.name = m[1]
+      part.filename = self._fileName(headerValue)
+    else if headerField == 'content-type'
+      part.mime = headerValue
+    else if headerField == 'content-transfer-encoding'
+      part.transferEncoding = headerValue.toLowerCase()
+    headerField = ''
+    headerValue = ''
+
+  parser.onHeadersEnd = () ->
+    switch part.transferEncoding
+      
+      when 'binary', '7bit', '8bit'
+        parser.onPartData = (b, start, end) ->
+          part.emit('data', b.slice(start, end))
+        parser.onPartEnd = () ->
+          part.emit('end')
+      
+      when 'base64'
+        parser.onPartData = (b, start, end) ->
+          part.transferBuffer += b.slice(start, end).toString('ascii')
+
+          ###
+           four bytes (chars) in base64 converts to three bytes in binary
+           encoding. So we should always work with a number of bytes that
+           can be divided by 4, it will result in a number of bytes that
+           can be divided vy 3.
+          ###
+
+          offset = parseInt(part.transferBuffer.length / 4, 10) * 4
+          part.emit('data', new Buffer(part.transferBuffer.substring(0, offset), 'base64'))
+          part.transferBuffer = part.transferBuffer.substring(offset)
+
+        parser.onPartEnd = () ->
+          part.emit('data', new Buffer(part.transferBuffer, 'base64'))
+          part.emit('end')
+
+      else
+        return new Error('unknown transfer-encoding')
+    handlePart(part)
+
+  parser.onEnd = () ->
+    ended = true
+
+  handlePart = (part) ->
+    fieldsSize = 0
+    if part.filename == undefined
+      value = ''
+      decoder = new StringDecoder(encoding)
+      part.on 'data', (buffer) ->
+        fieldsSize += buffer.length
+        if fieldsSize > maxFieldsSize
+          logger.error('maxFieldsSize exceeded, received ' + fieldsSize + ' bytes of field data')
+          return
+        value += decoder.write(buffer)
+      return
+    writableStreamBuffer = new (streamBuffers.WritableStreamBuffer)(
+      initialSize: 100 * 1024
+      incrementAmount: 10 * 1024
+    )
+    part.on 'data', (buffer) ->
+      if buffer.length == 0
+        return
+      writableStreamBuffer.write(buffer)
+    part.on 'end', ->
+      streamBufferList.push
+        streamBuffer: writableStreamBuffer
+        mime: part.mime
+        contentDescription: part.headers['content-description']
+        contentId: part.headers['content-id']
+        objectId: part.headers['object-id']
+
+  parser.initWithBoundary _multipartBoundary
+  parser.streamBufferList = streamBufferList
+  parser
+
+module.exports.parseMultipart = parseMultipart

lib/utils/normalizeUrl.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+urlUtil = require('url')
+
+
+# Returns a valid url for use with RETS server. If target url just contains a path, fullURL's protocol and host will be utilized.
+normalizeUrl = (targetUrl, fullUrl) ->
+  loginUrlObj = urlUtil.parse(fullUrl, true, true)
+  targetUrlObj = urlUtil.parse(targetUrl, true, true)
+  if targetUrlObj.host == loginUrlObj.host
+    return targetUrl
+  fixedUrlObj =
+    protocol: loginUrlObj.protocol
+    slashes: true
+    host: loginUrlObj.host
+    pathname: targetUrlObj.pathname
+    query: targetUrlObj.query
+  urlUtil.format(fixedUrlObj)
+
+  
+module.exports = normalizeUrl

lib/utils/queryOptions.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+
+# need to make a new object as we merge, as we don't want to modify the user's object
+mergeOptions = (options1, options2) ->
+  if !options1
+    return options2
+  result = {}
+  
+  # copy in options2 first, letting values from options1 overwrite and have priority
+  for own key of options2
+    result[key] = options2[key]
+  for own key of options1
+    result[key] = options1[key]
+    
+  result
+
+
+# default query parameters
+_queryOptionsDefaults =
+  queryType: 'DMQL2'
+  format: 'COMPACT-DECODED'
+  count: 1
+  standardNames: 0
+  restrictedIndicator: '***'
+  limit: 'NONE'
+
+
+normalizeOptions = (queryOptions) ->
+  if !queryOptions
+    throw new Error('queryOptions is required.')
+  if !queryOptions.searchType
+    throw new Error('searchType is required (ex: Property')
+  if !queryOptions.class
+    throw new Error('class is required (ex: RESI)')
+  if !queryOptions.query
+    throw new Error('query is required (ex: (MatrixModifiedDT=2014-01-01T00:00:00.000+) )')
+  mergeOptions(queryOptions, _queryOptionsDefaults)
+
+
+module.exports =
+  mergeOptions: mergeOptions
+  normalizeOptions: normalizeOptions

lib/utils/replyCodes.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+
+# Tags & Constant representation for reply codes
+
+# authoritative documentation for all reply codes in current rets standard:
+# http://www.reso.org/assets/RETS/Specifications/rets_1_8.pdf
+
+
+codeTagMap = {}
+
+# for readability, codes are presented in tag-code format, but we need to map them into code-tag format
+# using multiple calls to this helper because some tag names are NOT globally unique
+_registerCodes = (tagCodeMap) ->
+  for k, v of tagCodeMap
+    codeTagMap[v] = k
+
+
+_registerCodes
+  OPERATION_SUCCESSFUL: 0
+
+_registerCodes
+  SYSTEM_ERROR: 10000
+
+_registerCodes
+  ZERO_BALANCE:                       20003
+  BROKER_CODE_REQUIRED:               20012
+  BROKER_CODE_INVALID:                20013
+  DUPLICATE_LOGIN_PROHIBITED:         20022
+  MISC_LOGIN_ERROR:                   20036
+  CLIENT_AUTHENTICATION_FAILED:       20037
+  USER_AGENT_AUTHENTICATION_REQUIRED: 20041
+  SERVER_TEMPORARILY_DISABLED:        20050
+
+_registerCodes
+  INSECURE_PASSWORD_DISALLOWED:  20140
+  DUPLICATE_PASSWORD_DISALLOWED: 20141
+  ENCRYPTED_USERNAME_INVALID:    20142
+
+_registerCodes
+  UNKNOWN_QUERY_FIELD:     20200
+  NO_RECORDS_FOUND:        20201
+  INVALID_SELECT:          20202
+  MISC_SEARCH_ERROR:       20203
+  INVALID_QUERY_SYNTAX:    20206
+  UNAUTHORIZED_QUERY:      20207
+  MAX_RECORDS_EXCEEDED:    20208
+  TIMEOUT:                 20209
+  TOO_MANY_ACTIVE_QUERIES: 20210
+  QUERY_TOO_COMPLEX:       20211
+  INVALID_KEY_REQUEST:     20212
+  INVALID_KEY:             20213
+
+_registerCodes
+  INVALID_PARAMETER:          20301
+  RECORD_SAVE_ERROR:          20302
+  MISC_UPDATE_ERROR:          20303
+  WARNING_RESPONSE_NOT_GIVEN: 20311
+  WARNING_RESPONSE_GIVEN:     20312
+
+_registerCodes
+  INVALID_RESOURCE:         20400
+  INVALID_OBJECT_TYPE:      20401
+  INVALID_IDENTIFIER:       20402
+  NO_OBJECT_FOUND:          20403
+  UNSUPPORTED_MIME_TYPE:    20406
+  UNAUTHORIZED_RETRIEVAL:   20407
+  RESOURCE_UNAVAILABLE:     20408
+  OBJECT_UNAVAILABLE:       20409
+  REQUEST_TOO_LARGE:        20410
+  TIMEOUT:                  20411
+  TOO_MANY_ACTIVE_REQUESTS: 20412
+  MISC_ERROR:               20413
+
+_registerCodes
+  INVALID_RESOURCE:         20500
+  INVALID_METADATA_TYPE:    20501
+  INVALID_IDENTIFIER:       20502
+  NO_METADATA_FOUND:        20503
+  UNSUPPORTED_MIME_TYPE:    20506
+  UNAUTHORIZED_RETRIEVAL:   20507
+  RESOURCE_UNAVAILABLE:     20508
+  METADATA_UNAVAILABLE:     20509
+  REQUEST_TOO_LARGE:        20510
+  TIMEOUT:                  20511
+  TOO_MANY_ACTIVE_REQUESTS: 20512
+  MISC_ERROR:               20513
+  DTD_VERSION_UNAVAIL:      20514
+
+_registerCodes
+  NOT_LOGGED_IN:          20701
+  MISC_TRANSACTION_ERROR: 20702
+
+_registerCodes
+  UNKNOWN_RESOURCE:                20800
+  INVALID_OBJECT_TYPE:             20801
+  INVALID_IDENTIFIER:              20802
+  INVALID_UPDATE_ACTION:           20803
+  INCONSISTENT_REQUEST_PARAMETERS: 20804
+  DELETE_TARGET_NOT_FOUND:         20805
+  UNSUPPORTED_MIME_TYPE:           20806
+  UNAUTHORIZED:                    20807
+  SOME_OBJECTS_NOT_DELETED:        20808
+  BUSINESS_RULES_VIOLATION:        20809
+  FILE_TOO_LARGE:                  20810
+  TIMEOUT:                         20811
+  TOO_MANY_ACTIVE_REQUESTS:        20812
+  MISC_ERROR:                      20813
+
+
+module.exports =
+  tagMap: codeTagMap
+  getReplyTag: (code) -> codeTagMap[code]

lib/utils/retsHttp.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+Promise = require('bluebird')
+logger = require('winston')
+expat = require('node-expat')
+
+errors = require('./errors')
+
+
+callRetsMethod = (methodName, retsSession, queryOptions) ->
+  ###
+  Promise.resolve
+    body: require('fs').readFileSync("/Users/joe/work/realtymaps/tmp/dump_#{methodName}_#{queryOptions.offset||0}.xml")
+    response: {"headers":{"rets-version":"RETS/1.7.2","server":"nginx/1.6.0"}}
+  ###
+  logger.debug("RETS #{methodName}", queryOptions)
+  Promise.try () ->
+    retsSession(qs: queryOptions)
+  .catch (error) ->
+    logger.debug "RETS #{methodName} error:\n" + JSON.stringify(error)
+    Promise.reject(error)
+  .spread (response, body) ->
+    if response.statusCode != 200
+      error = new errors.RetsServerError(methodName, response.statusCode, response.statusMessage)
+      logger.debug "RETS #{methodName} error:\n" + error.message
+      return Promise.reject(error)
+    body: body
+    response: response
+
+
+streamRetsMethod = (methodName, retsSession, queryOptions, failCallback) ->
+  #require('fs').createReadStream("/Users/joe/work/realtymaps/tmp/dump_#{methodName}_#{queryOptions.offset||0}.xml")
+  logger.debug("RETS #{methodName} stream", queryOptions)
+  done = false
+  errorHandler = (error) ->
+    if done
+      return
+    done = true
+    logger.debug "RETS #{methodName} error:\n" + JSON.stringify(error)
+    failCallback(error)
+  responseHandler = (response) ->
+    if done
+      return
+    done = true
+    if response.statusCode != 200
+      error = new errors.RetsServerError('search', response.statusCode, response.statusMessage)
+      logger.debug "RETS #{methodName} error:\n" + error.message
+      failCallback(error)
+  stream = retsSession(qs: queryOptions)
+  stream.on 'error', errorHandler
+  stream.on 'response', responseHandler
+
+
+module.exports =
+  callRetsMethod: callRetsMethod
+  streamRetsMethod: streamRetsMethod

lib/utils/retsParsing.coffee

+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
+
+expat = require('node-expat')
+through2 = require('through2')
+
+errors = require('./errors')
+replyCodes = require('./replyCodes')
+hex2a = require('./hex2a')
+
+
+# Parsing as performed here and in the other modules of this project relies on some simplifying assumptions.  DO NOT
+# COPY OR MODIFY THIS LOGIC BLINDLY!  It works correctly for well-formed XML which adheres to the RETS specifications,
+# and does not attempt to check for or properly handle XML not of that form.  In particular, it does not keep track of
+# the element stack to ensure elements (and text) are found only in the expected locations.
+
+
+# a parser with some basic common functionality, intended to be extended for real use
+getSimpleParser = (errCallback) ->
+  result =
+    currElementName: null
+    parser: new expat.Parser('UTF-8')
+    finish: () ->
+      #result.parser.stop()
+      result.parser.removeAllListeners()
+    status: null
+
+  result.parser.once 'startElement', (name, attrs) ->
+    if name != 'RETS'
+      result.finish()
+      return errCallback(new Error('Unexpected results. Please check the RETS URL.'))
+
+  result.parser.on 'startElement', (name, attrs) ->
+    result.currElementName = name
+    if name != 'RETS' && name != 'RETS-STATUS'
+      return
+    result.status = attrs
+    if attrs.ReplyCode != '0' && attrs.ReplyCode != '20208'
+      result.finish()
+      return errCallback(new errors.RetsReplyError(attrs.ReplyCode, attrs.ReplyText))
+
+  result.parser.on 'error', (err) ->
+    result.finish()
+    errCallback(new Error("XML parsing error: #{err}"))
+
+  result.parser.on 'end', () ->
+    result.finish()
+    errCallback(new Error("Unexpected end of xml stream."))
+
+  return result
+
+
+# parser that deals with column/data tags, as returned for metadata and search queries
+getStreamParser = (metadataTag, rawData) ->
+  if metadataTag
+    rawData = false
+    result =
+      rowsReceived: 0
+      entriesReceived: 0
+    delimiter = '\t'
+  else
+    result =
+      rowsReceived: 0
+      maxRowsExceeded: false
+    delimiter = null
+  columnText = null
+  dataText = null
+  columns = null
+  currElementName = null
+
+  parser = new expat.Parser('UTF-8')
+  retsStream = through2.obj()
+  finish = (type, payload) ->
+    parser.removeAllListeners()
+    retsStream.write(type: type, payload: payload)
+    retsStream.end()
+  fail = (err) ->
+    finish('error', err)
+  writeOutput = (type, payload) ->
+    retsStream.write(type: type, payload: payload)
+  processStatus = (attrs) ->
+    if attrs.ReplyCode != '0' && attrs.ReplyCode != '20208'
+      return fail(new errors.RetsReplyError(attrs.ReplyCode, attrs.ReplyText))
+    status =
+      replyCode: attrs.ReplyCode
+      replyTag: replyCodes.tagMap[attrs.ReplyCode]
+      replyText: attrs.ReplyText
+    writeOutput('status', status)
+
+  parser.once 'startElement', (name, attrs) ->
+    if name != 'RETS'
+      return fail(new Error('Unexpected results. Please check the RETS URL.'))
+    processStatus(attrs)
+  
+    parser.on 'startElement', (name, attrs) ->
+      currElementName = name
+      switch name
+        when 'DATA'
+          dataText = ''
+        when 'COLUMNS'
+          columnText = ''
+        when metadataTag
+          writeOutput('metadataStart', attrs)
+          result.rowsReceived = 0
+        when 'COUNT'
+          writeOutput('count', parseInt(attrs.Records))
+        when 'MAXROWS'
+          result.maxRowsExceeded = true
+        when 'DELIMITER'
+          delimiter = hex2a(attrs.value)
+          writeOutput('delimiter', delimiter)
+        when 'RETS-STATUS'
+          processStatus(attrs)
+
+  parser.on 'text', (text) ->
+    switch currElementName
+      when 'DATA'
+        dataText += text
+      when 'COLUMNS'
+        columnText += text
+
+  if rawData
+    parser.on 'endElement', (name) ->
+      currElementName = null
+      switch name
+        when 'DATA'
+          writeOutput('data', dataText)
+          result.rowsReceived++
+        when 'COLUMNS'
+          writeOutput('columns', columnText)
+        when 'RETS'
+          finish('done', result)
+  else
+    parser.on 'endElement', (name) ->
+      currElementName = null
+      switch name
+        when 'DATA'
+          if !columns
+            return fail(new Error('Failed to parse columns'))
+          data = dataText.split(delimiter)
+          model = {}
+          i=1
+          while i < columns.length-1
+            model[columns[i]] = data[i]
+            i++
+          writeOutput('data', model)
+          result.rowsReceived++
+        when 'COLUMNS'
+          if !delimiter
+            return fail(new Error('Failed to parse delimiter'))
+          columns = columnText.split(delimiter)
+          writeOutput('columns', columns)
+        when metadataTag
+          result.entriesReceived++
+          writeOutput('metadataEnd', result.rowsReceived)
+        when 'RETS'
+          if metadataTag
+            delete result.rowsReceived
+          finish('done', result)
+
+  parser.on 'error', (err) ->
+    fail(new Error("XML parsing error: #{err.stack}"))
+  
+  parser.on 'end', () ->
+    # we remove event listeners upon success, so getting here implies failure
+    fail(new Error("Unexpected end of xml stream."))
+  
+  parser: parser
+  fail: fail
+  retsStream: retsStream
+  
+  
+module.exports =
+  getSimpleParser: getSimpleParser
+  getStreamParser: getStreamParser

index.js

@@ -1,33 +0,9 @@
-var Promise = require('bluebird');
+/* jshint node:true */
+/* jshint -W097 */
+'use strict';
+
+
 var coffee = require('coffee-script');
 coffee.register();
 
-var replycodes = require('./lib/replycodes');
-var Client = require('./lib/client');
-var utils = require('./lib/utils');
-
-/*  Available settings:
- *      loginUrl: RETS login URL (i.e http://<MLS_DOMAIN>/rets/login.ashx)
- *      username: username credential
- *      password: password credential
- *      version: rets version
- *
- *      //RETS-UA-Authorization
- *      userAgent
- *      userAgentPassword
- *      sessionId
- */
-module.exports = {
-  replycode: replycodes.codeMap,
-  RetsReplyError: utils.RetsReplyError,
-  RetsServerError: utils.RetsServerError,
-  Client: Client,
-  getAutoLogoutClient: function(settings, handler) {
-    var client = new Client(settings);
-    return client.login()
-    .then(handler)
-    .finally(function() {
-      return client.logout();
-    });
-  }
-};
+module.exports = require('./lib/api');

lib/client.coffee

@@ -0,1 +1,4 @@
+### jshint node:true ###
+### jshint -W097 ###
+'use strict'
 
@@ -6,8 +9,10 @@ crypto = require('crypto')
 
-auth = require('./auth')
-metadata = require('./metadata')
-search = require('./search')
-object = require('./object')
-appUtils = require('./utils')
+metadata = require('./clientModules/metadata')
+search = require('./clientModules/search')
+object = require('./clientModules/object')
 
+auth = require('./utils/auth')
+normalizeUrl = require('./utils/normalizeUrl')
+
+
 URL_KEYS =
@@ -64,3 +69,3 @@ GET_METADATA: "GetMetadata"
         if @systemData[val]
-          @urls[val] = appUtils.getValidUrl(@systemData[val], @settings.loginUrl)
+          @urls[val] = normalizeUrl(@systemData[val], @settings.loginUrl)
 
@@ -78,2 +83,13 @@ @metadata = metadata(@baseRetsSession.defaults(uri: @urls[URL_KEYS.GET_METADATA]))
 
+
+Client.getAutoLogoutClient = (settings, handler) -> Promise.try () ->
+  client = new Client(settings)
+  client.login()
+  .then () ->
+    Promise.try () ->
+      handler(client)
+    .finally () ->
+      client.logout()
+
+
 module.exports = Client

package.json

 {
   "name": "rets-client",
-  "version": "2.0.7",
+  "version": "3.0.0",
   "description": "A RETS client (Real Estate Transaction Standard).",
@@ -11,7 +11,8 @@ "main": "index.js",
     "formidable": "~1.0.15",
+    "node-expat": "^2.3.10",
     "request": "^2.55.0",
     "stream-buffers": "~0.2.5",
+    "through2": "^2.0.0",
     "tough-cookie": "^0.13.0",
-    "winston": "~0.7.3",
-    "xml2js": "~0.4.4"
+    "winston": "~0.7.3"
   },
@@ -34,3 +35,6 @@ "directories": {
     "promise",
-    "mls"
+    "mls",
+    "stream",
+    "streams",
+    "streaming"
   ],
@@ -42,3 +46,3 @@ "author": "Steve Bruno <[e-mail]>",
   ],
-  "license": "The MIT License (MIT)",
+  "license": "MIT",
   "bugs": {
@@ -45,0 +49,0 @@ "url": "https://github.com/sbruno81/rets-client/issues"

README.md

@@ -5,9 +5,27 @@ rets-client
 
-Version 2.x of rets-client has a completely different interface from the 1.x version -- code written for 1.x will not
-work with 2.x.  If you wish to continue to use the 1.x version, you can use the
-[v1 branch](https://github.com/sbruno81/rets-client/tree/v1).
+## Changes
 
-This interface uses promises, and future development plans include an optional stream-based interface
-for better performance with large datasets and/or large objects.
+Version 3.x is out!  This represents a substantial rewrite of the underlying code, which should improve performance
+(both CPU and memory use) for almost all RETS calls by using node-expat instead of xml2js for xml parsing.  The changes
+are mostly internal, however there is 1 small backward-incompatible change needed for correctness, described below.
+The large internal refactor plus even a small breaking change warrants a major version bump.
 
+Many of the metadata methods are capable of returning multiple sets of data, including (but not limited to) the
+getAll* methods.  Versions 1.x and 2.x did not handle this properly; version 1.x returned the values from the last set
+encountered, and version 2.x returned the values from the first set encountered.  Version 3.x always returns all values
+encountered, by returning an array of data sets rather than a single one.
+
+In addition to the methods available in 2.x, version 3.0 adds `client.search.stream.searchRets()`, which returns a
+text stream of the raw XML result, and `client.search.stream.query()`, which returns a stream of low-level objects
+parsed from the XML.  (See the [streaming example](#simple-streaming-example) below.)  These streams, if used properly,
+should result in a much lower memory footprint than their corresponding non-streaming counterparts.
+
+Version 3.x has almost the same interface as 2.x, which is completely different from 1.x.  If you wish to continue to
+use the 1.x version, you can use the [v1 branch](https://github.com/sbruno81/rets-client/tree/v1).
+
+## Implementation Notes
+
+This interface uses promises, and an optional stream-based interface for better performance with large search results.
+Future development will include an optional stream-based interface for large objects.
+
 This library is written primarily in CoffeeScript, but may be used just as easily in a Node app using Javascript or
@@ -28,3 +46,3 @@ CoffeeScript.  Promises in this module are provided by [Bluebird](https://github.com/petkaantonov/bluebird).
 #### TODO
-- create optional streaming interface 
+- create optional streaming interface for object downloads; when implemented, this will make version 3.1
 - create unit tests -- specifically ones that run off example RETS data rather than requiring access to a real RETS server
@@ -79,5 +97,5 @@
         outputFields(data, ['Version', 'Date']);
-        for (var dataItem = 0; dataItem < data.results.length; dataItem++) {
+        for (var dataItem = 0; dataItem < data.results[0].metadata.length; dataItem++) {
           console.log("-------- Resource " + dataItem + " --------");
-          outputFields(data.results[dataItem], ['ResourceID', 'StandardName', 'VisibleName', 'ObjectVersion']);
+          outputFields(data.results[0].metadata[dataItem], ['ResourceID', 'StandardName', 'VisibleName', 'ObjectVersion']);
         }
@@ -92,5 +110,5 @@ }).then(function () {
         outputFields(data, ['Version', 'Date', 'Resource']);
-        for (var classItem = 0; classItem < data.results.length; classItem++) {
+        for (var classItem = 0; classItem < data.results[0].metadata.length; classItem++) {
           console.log("-------- Table " + classItem + " --------");
-          outputFields(data.results[classItem], ['ClassName', 'StandardName', 'VisibleName', 'TableVersion']);
+          outputFields(data.results[0].metadata[classItem], ['ClassName', 'StandardName', 'VisibleName', 'TableVersion']);
         }
@@ -105,7 +123,7 @@ }).then(function () {
         outputFields(data, ['Version', 'Date', 'Resource', 'Class']);
-        for (var tableItem = 0; tableItem < data.results.length; tableItem++) {
+        for (var tableItem = 0; tableItem < data.results[0].metadata.length; tableItem++) {
           console.log("-------- Field " + tableItem + " --------");
-          outputFields(data.results[tableItem], ['MetadataEntryID', 'SystemName', 'ShortName', 'LongName', 'DataType']);
+          outputFields(data.results[0].metadata[tableItem], ['MetadataEntryID', 'SystemName', 'ShortName', 'LongName', 'DataType']);
         }
-        return data.results
+        return data.results[0].metadata
       }).then(function (fieldsData) {
@@ -119,3 +137,3 @@ var plucked = [];
         //perform a query using DQML2 -- pass resource, class, and query, and options
-        return client.search.query("OpenHouse", "OPENHOUSE", "(OpenHouseType=PUBLIC),(ActiveYN=1)", {limit:100, offset:1})
+        return client.search.query("OpenHouse", "OPENHOUSE", "(OpenHouseType=PUBLIC),(ActiveYN=1)", {limit:100, offset:10})
         .then(function (searchData) {
@@ -153,2 +171,45 @@ console.log("===========================================");
   });
-```
+```
+
+#### Simple streaming example
+```javascript
+  var rets = require('rets-client');
+  var through2 = require('through2');
+  var Promise = require('bluebird');
+  // establish connection to RETS server which auto-logs out when we're done
+  rets.getAutoLogoutClient(clientSettings, function (client) {
+    // in order to have the auto-logout function work properly, we need to make a promise that either rejects or
+    // resolves only once we're done processing the stream
+    return new Promise(function (reject, resolve) {
+      var retsStream = client.search.stream.query("OpenHouse", "OPENHOUSE", "(OpenHouseType=PUBLIC),(ActiveYN=1)", {limit:100, offset:10});
+      var processorStream = through2.obj(function (event, encoding, callback) {
+        switch (event.type) {
+          case 'data':
+            // event.payload is an object representing a single row of results
+            // make sure callback is called only when all processing is complete
+            doAsyncProcessing(event.payload, callback);
+            break;
+          case 'done':
+            // event.payload is an object containing a count of rows actually received, plus some other things
+            // now we can resolve the auto-logout promise
+            resolve(event.payload.rowsReceived);
+            callback();
+            break;
+          case 'error':
+            // event.payload is an Error object
+            console.log('Error streaming RETS results: '+event.payload);
+            retsStream.unpipe(processorStream);
+            processorStream.end();
+            // we need to reject the auto-logout promise
+            reject(event.payload);
+            callback();
+            break;
+          default:
+            // ignore other events
+            callback();
+        }
+      });
+      retsStream.pipe(processorStream);
+    });
+  });
+```

lib/auth.coffee

-### jshint node:true ###
-### jshint -W097 ###
-'use strict'
-
-logger = require('winston')
-Promise = require('bluebird')
-xmlParser = Promise.promisify(require('xml2js').parseString)
-
-utils = require('./utils')
-
-
-###
-# Executes RETS login routine.
-###
-
-login = (retsSession) ->
-  logger.debug 'RETS method login'
-  utils.callRetsMethod('login', Promise.promisify(retsSession), {})
-  .then (result) ->
-    xmlParser(result.body)
-    .then (parsed) ->
-      if !parsed || !parsed.RETS
-        throw new Error('Unexpected results. Please check the RETS URL')
-      keyVals = parsed.RETS['RETS-RESPONSE'][0].split('\r\n')
-      systemData = {}
-      for keyVal in keyVals
-        split = keyVal.split('=')
-        if split.length > 1
-          systemData[split[0]] = split[1]
-      systemData.retsVersion = result.response.headers['rets-version']
-      systemData.retsServer = result.response.headers.server
-      systemData
-
-
-###
-# Logouts RETS user
-###
-
-logout = (retsSession) ->
-  logger.debug 'RETS method logout'
-  utils.callRetsMethod('logout', Promise.promisify(retsSession), {})
-  .then (result) ->
-    logger.debug 'Logout success'
-
-
-module.exports =
-  login: login
-  logout: logout

lib/metadata.coffee

-### jshint node:true ###
-### jshint -W097 ###
-'use strict'
-
-logger = require('winston')
-Promise = require('bluebird')
-xmlParser = Promise.promisify(require('xml2js').parseString)
-
-utils = require('./utils')
-
-
-_getParsedMetadataFactory = (retsSession, type, format='COMPACT') ->
-  (id) -> Promise.try () ->
-    if !id
-      throw new Error('Resource type id is required (or for some types of metadata, "0" retrieves for all resource types)')
-    options =
-      Type: type
-      Id: id
-      Format: format
-    logger.debug('RETS getMetadata', options)
-    utils.callRetsMethod('getMetadata', retsSession, options)
-    .then (result) ->
-      utils.parseCompact(result.body, type)
-
-
-_getParsedAllMetadataFactory = (retsSession, type, format='COMPACT') ->
-  options =
-    Type: type
-    Id: '0'
-    Format: format
-  () -> Promise.try () ->
-    logger.debug('RETS getMetadata', options)
-    utils.callRetsMethod('getMetadata', retsSession, options)
-    .then (result) ->
-      utils.parseCompact(result.body, type)
-
-
-###
-# Retrieves RETS Metadata.
-#
-# @param type Metadata type (i.e METADATA-RESOURCE, METADATA-CLASS)
-# @param id Metadata id
-# @param format Data format (i.e. COMPACT, COMPACT-DECODED), defaults to 'COMPACT'
-###
-
-getMetadata = (type, id, format='COMPACT') -> Promise.try () =>
-  logger.debug('RETS getMetadata', type, id, format)
-  if !type
-    throw new Error('Metadata type is required')
-  if !id
-    throw new Error('Resource type id is required (or for some types of metadata, "0" retrieves for all resource types)')
-  options =
-    Type: type
-    Id: id
-    Format: format
-  utils.callRetsMethod('getMetadata', @retsSession, options)
-  .then (result) ->
-    result.body
-
-
-###
-# Helper that retrieves RETS system metadata
-###
-
-getSystem = () ->
-  @getMetadata('METADATA-SYSTEM')
-  .then xmlParser
-  .then utils.replyCodeCheck
-  .then (parsedXml) ->
-    systemData = result.RETS['METADATA-SYSTEM']?[0]
-    if !systemData
-      throw new Error("Failed to parse system XML: #{systemData}")
-    metadataVersion: systemData.$.Version
-    metadataDate: systemData.$.Date
-    systemId: systemData.SYSTEM[0].$.SystemID
-    systemDescription: systemData.SYSTEM[0].$.SystemDescription
-
-
-module.exports = (_retsSession) ->
-  _retsSession = Promise.promisify(_retsSession)
-  if !_retsSession
-    throw new Error('System data not set; invoke login().')
-  retsSession: _retsSession
-  getMetadata: getMetadata
-  getSystem: getSystem
-  getResources:       _getParsedMetadataFactory(_retsSession, 'METADATA-RESOURCE')
-  getAllForeignKeys:  _getParsedAllMetadataFactory(_retsSession, 'METADATA-FOREIGNKEYS')
-  getForeignKeys:     _getParsedMetadataFactory(_retsSession, 'METADATA-FOREIGNKEYS')
-  getAllClass:        _getParsedAllMetadataFactory(_retsSession, 'METADATA-CLASS')
-  getClass:           _getParsedMetadataFactory(_retsSession, 'METADATA-CLASS')
-  getAllTable:        _getParsedAllMetadataFactory(_retsSession, 'METADATA-TABLE')
-  getTable:           _getParsedMetadataFactory(_retsSession, 'METADATA-TABLE')
-  getAllLookups:      _getParsedAllMetadataFactory(_retsSession, 'METADATA-LOOKUP')
-  getLookups:         _getParsedMetadataFactory(_retsSession, 'METADATA-LOOKUP')
-  getAllLookupTypes:  _getParsedAllMetadataFactory(_retsSession, 'METADATA-LOOKUP_TYPE')
-  getLookupTypes:     _getParsedMetadataFactory(_retsSession, 'METADATA-LOOKUP_TYPE')
-  getObject:          _getParsedMetadataFactory(_retsSession, 'METADATA-OBJECT')

lib/multipart.coffee

-### jshint node:true ###
-### jshint -W097 ###
-'use strict'
-
-logger = require('winston')
-MultipartParser = require('formidable/lib/multipart_parser').MultipartParser
-Stream = require('stream').Stream
-StringDecoder = require('string_decoder').StringDecoder
-streamBuffers = require('stream-buffers')
-Promise = require('bluebird')
-#Multipart parser derived from formidable library. See https://github.com/felixge/node-formidable
-
-
-parseMultipart = (buffer, _multipartBoundary) -> Promise.try () ->
-  parser = getParser(_multipartBoundary)
-  if parser instanceof Error
-    return Promise.reject(parser)
-  parser.write(buffer)
-  dataBufferList = []
-  for streamBuffer in parser.streamBufferList
-    dataBufferList.push
-      buffer: streamBuffer.streamBuffer.getContents()
-      mime: streamBuffer.mime
-      description: streamBuffer.description
-      contentDescription: streamBuffer.contentDescription
-      contentId: streamBuffer.contentId
-      objectId: streamBuffer.objectId
-  dataBufferList
-
-
-getParser = (_multipartBoundary) ->
-  streamBufferList = []
-  parser = new MultipartParser()
-  headerField = ''
-  headerValue = ''
-  part = {}
-  encoding = 'utf8'
-  ended = false
-  maxFields = 1000
-  maxFieldsSize = 2 * 1024 * 1024
-
-  parser.onPartBegin = () ->
-    part = new Stream()
-    part.readable = true
-    part.headers = {}
-    part.name = null
-    part.filename = null
-    part.mime = null
-    part.transferEncoding = 'binary'
-    part.transferBuffer = ''
-    headerField = ''
-    headerValue = ''
-
-  parser.onHeaderField = (b, start, end) ->
-    headerField += b.toString(encoding, start, end)
-
-  parser.onHeaderValue = (b, start, end) ->
-    headerValue += b.toString(encoding, start, end)
-
-  parser.onHeaderEnd = () =>
-    headerField = headerField.toLowerCase()
-    part.headers[headerField] = headerValue
-    if headerField == 'content-disposition'
-      m = headerValue.match(/\bname="([^"]+)"/i)
-      if m
-        part.name = m[1]
-      part.filename = self._fileName(headerValue)
-    else if headerField == 'content-type'
-      part.mime = headerValue
-    else if headerField == 'content-transfer-encoding'
-      part.transferEncoding = headerValue.toLowerCase()
-    headerField = ''
-    headerValue = ''
-
-  parser.onHeadersEnd = () ->
-    switch part.transferEncoding
-      
-      when 'binary', '7bit', '8bit'
-        parser.onPartData = (b, start, end) ->
-          part.emit('data', b.slice(start, end))
-        parser.onPartEnd = () ->
-          part.emit('end')
-      
-      when 'base64'
-        parser.onPartData = (b, start, end) ->
-          part.transferBuffer += b.slice(start, end).toString('ascii')
-
-          ###
-           four bytes (chars) in base64 converts to three bytes in binary
-           encoding. So we should always work with a number of bytes that
-           can be divided by 4, it will result in a number of bytes that
-           can be divided vy 3.
-          ###
-
-          offset = parseInt(part.transferBuffer.length / 4, 10) * 4
-          part.emit('data', new Buffer(part.transferBuffer.substring(0, offset), 'base64'))
-          part.transferBuffer = part.transferBuffer.substring(offset)
-
-        parser.onPartEnd = () ->
-          part.emit('data', new Buffer(part.transferBuffer, 'base64'))
-          part.emit('end')
-
-      else
-        return new Error('unknown transfer-encoding')
-    handlePart(part)
-
-  parser.onEnd = () ->
-    ended = true
-
-  handlePart = (part) ->
-    fieldsSize = 0
-    if part.filename == undefined
-      value = ''
-      decoder = new StringDecoder(encoding)
-      part.on 'data', (buffer) ->
-        fieldsSize += buffer.length
-        if fieldsSize > maxFieldsSize
-          logger.error('maxFieldsSize exceeded, received ' + fieldsSize + ' bytes of field data')
-          return
-        value += decoder.write(buffer)
-      return
-    writableStreamBuffer = new (streamBuffers.WritableStreamBuffer)(
-      initialSize: 100 * 1024
-      incrementAmount: 10 * 1024
-    )
-    part.on 'data', (buffer) ->
-      if buffer.length == 0
-        return
-      writableStreamBuffer.write(buffer)
-    part.on 'end', ->
-      streamBufferList.push
-        streamBuffer: writableStreamBuffer
-        mime: part.mime
-        contentDescription: part.headers['content-description']
-        contentId: part.headers['content-id']
-        objectId: part.headers['object-id']
-
-  parser.initWithBoundary _multipartBoundary
-  parser.streamBufferList = streamBufferList
-  parser
-
-module.exports.parseMultipart = parseMultipart

lib/object.coffee

-### jshint node:true ###
-### jshint -W097 ###
-'use strict'
-
-logger = require('winston')
-streamBuffers = require('stream-buffers')
-Promise = require('bluebird')
-
-multipart = require('./multipart')
-
-
-###
-# Retrieves RETS object data.
-#
-# @param resourceType Rets resource type (ex: Property)
-# @param objectType Rets object type (ex: LargePhoto)
-# @param objectId Object identifier
-###
-
-getObject = (resourceType, objectType, objectId) ->
-  logger.debug 'RETS method getObject'
-  if !resourceType
-    throw new Error('Resource type id is required')
-  if !objectType
-    throw new Error('Object type id is required')
-  if !objectId
-    throw new Error('Object id is required')
-  options =
-    Type: objectType
-    Id: objectId
-    Resource: resourceType
-  
-  # prepare stream buffer for object data
-  writableStreamBuffer = new (streamBuffers.WritableStreamBuffer)(
-    initialSize: 100 * 1024
-    incrementAmount: 10 * 1024)
-  req = @retsSession(options)
-  
-  #pipe object data to stream buffer
-  new Promise (resolve, reject) ->
-    req.pipe(writableStreamBuffer)
-    req.on('error', reject)
-    contentType = null
-    req.on 'response', (_response) ->
-      contentType = _response.headers['content-type']
-    req.on 'end', ->
-      resolve
-        contentType: contentType
-        data: writableStreamBuffer.getContents()
-
-###
-# Helper that retrieves a list of photo objects.
-#
-# @param resourceType Rets resource type (ex: Property)
-# @param photoType Photo object type, based on getObjects meta call (ex: LargePhoto, Photo)
-# @param matrixId Photo matrix identifier.
-#
-# Each item in resolved data list is an object with the following data elements:
-#   buffer: <data buffer>,
-#   mime: <data buffer mime type>,
-#   description: <data description>,
-#   contentDescription: <data content description>,
-#   contentId: <content identifier>,
-#   objectId: <object identifier>
-###
-
-getPhotos = (resourceType, photoType, matrixId) ->
-  @getObject(resourceType, photoType, matrixId + ':*')
-  .then (result) ->
-    multipartBoundary = result.contentType.match(/boundary=(?:"([^"]+)"|([^;]+))/ig)[0].match(/[^boundary=^"]\w+[^"]/ig)[0]
-    if !multipartBoundary
-      throw new Error('Could not find multipart boundary')
-    multipart.parseMultipart(new Buffer(result.data), multipartBoundary)
-    .catch (err) ->
-      logger.error err
-      throw new Error('Error parsing multipart data')
-
-
-module.exports = (_retsSession) ->
-  if !_retsSession
-    throw new Error('System data not set; invoke login().')
-  retsSession: Promise.promisify(_retsSession)
-  getObject: getObject
-  getPhotos: getPhotos

lib/replycodes.coffee

-# Tags & Constant representation for reply codes
- 
-# authoritative documentation for all reply codes in current rets standard:
-# http://www.reso.org/assets/RETS/Specifications/rets_1_8.pdf
-
-
-codeTagMap = {}
-
-# for readability, codes are presented in tag-code format, but we need to map them into code-tag format for use
-# using multiple calls to this helper because some tags are NOT globally unique
-_registerCodes = (tagCodeMap) ->
-  for k, v of tagCodeMap
-    codeTagMap[v] = k
-
-
-_registerCodes
-  OPERATION_SUCCESSFUL: 0
-
-_registerCodes
-  SYSTEM_ERROR: 10000
-
-_registerCodes
-  ZERO_BALANCE:                       20003
-  BROKER_CODE_REQUIRED:               20012
-  BROKER_CODE_INVALID:                20013
-  DUPLICATE_LOGIN_PROHIBITED:         20022
-  MISC_LOGIN_ERROR:                   20036
-  CLIENT_AUTHENTICATION_FAILED:       20037
-  USER_AGENT_AUTHENTICATION_REQUIRED: 20041
-  SERVER_TEMPORARILY_DISABLED:        20050
-
-_registerCodes
-  INSECURE_PASSWORD_DISALLOWED:  20140
-  DUPLICATE_PASSWORD_DISALLOWED: 20141
-  ENCRYPTED_USERNAME_INVALID:    20142
-
-_registerCodes
-  UNKNOWN_QUERY_FIELD:     20200
-  NO_RECORDS_FOUND:        20201
-  INVALID_SELECT:          20202
-  MISC_SEARCH_ERROR:       20203
-  INVALID_QUERY_SYNTAX:    20206
-  UNAUTHORIZED_QUERY:      20207
-  MAX_RECORDS_EXCEEDED:    20208
-  TIMEOUT:                 20209
-  TOO_MANY_ACTIVE_QUERIES: 20210
-  QUERY_TOO_COMPLEX:       20211
-  INVALID_KEY_REQUEST:     20212
-  INVALID_KEY:             20213
-
-_registerCodes
-  INVALID_PARAMETER:          20301
-  RECORD_SAVE_ERROR:          20302
-  MISC_UPDATE_ERROR:          20303
-  WARNING_RESPONSE_NOT_GIVEN: 20311
-  WARNING_RESPONSE_GIVEN:     20312
-
-_registerCodes
-  INVALID_RESOURCE:         20400
-  INVALID_OBJECT_TYPE:      20401
-  INVALID_IDENTIFIER:       20402
-  NO_OBJECT_FOUND:          20403
-  UNSUPPORTED_MIME_TYPE:    20406
-  UNAUTHORIZED_RETRIEVAL:   20407
-  RESOURCE_UNAVAILABLE:     20408
-  OBJECT_UNAVAILABLE:       20409
-  REQUEST_TOO_LARGE:        20410
-  TIMEOUT:                  20411
-  TOO_MANY_ACTIVE_REQUESTS: 20412
-  MISC_ERROR:               20413
-
-_registerCodes
-  INVALID_RESOURCE:         20500
-  INVALID_METADATA_TYPE:    20501
-  INVALID_IDENTIFIER:       20502
-  NO_METADATA_FOUND:        20503
-  UNSUPPORTED_MIME_TYPE:    20506
-  UNAUTHORIZED_RETRIEVAL:   20507
-  RESOURCE_UNAVAILABLE:     20508
-  METADATA_UNAVAILABLE:     20509
-  REQUEST_TOO_LARGE:        20510
-  TIMEOUT:                  20511
-  TOO_MANY_ACTIVE_REQUESTS: 20512
-  MISC_ERROR:               20513
-  DTD_VERSION_UNAVAIL:      20514
-
-_registerCodes
-  NOT_LOGGED_IN:          20701
-  MISC_TRANSACTION_ERROR: 20702
-
-_registerCodes
-  UNKNOWN_RESOURCE:                20800
-  INVALID_OBJECT_TYPE:             20801
-  INVALID_IDENTIFIER:              20802
-  INVALID_UPDATE_ACTION:           20803
-  INCONSISTENT_REQUEST_PARAMETERS: 20804
-  DELETE_TARGET_NOT_FOUND:         20805
-  UNSUPPORTED_MIME_TYPE:           20806
-  UNAUTHORIZED:                    20807
-  SOME_OBJECTS_NOT_DELETED:        20808
-  BUSINESS_RULES_VIOLATION:        20809
-  FILE_TOO_LARGE:                  20810
-  TIMEOUT:                         20811
-  TOO_MANY_ACTIVE_REQUESTS:        20812
-  MISC_ERROR:                      20813
-
-
-module.exports =
-  tagMap: codeTagMap

lib/search.coffee

-### jshint node:true ###
-### jshint -W097 ###
-'use strict'
-
-logger = require('winston')
-Promise = require('bluebird')
-
-utils = require('./utils')
-
-
-setDefaults = (options, defaults) ->
-  if !options
-    return options
-  result = {}
-  for own key of options
-    result[key] = options[key]
-  for own key of defaults when key not of result
-    result[key] = defaults[key]
-  result
-
-
-#default query parameters
-queryOptionsDefaults =
-  queryType: 'DMQL2'
-  format: 'COMPACT-DECODED'
-  count: 1
-  standardNames: 0
-  restrictedIndicator: '***'
-  limit: 'NONE'
-
-###
-# Invokes RETS search operation.
-#
-# @param _queryOptions Search query options.
-#    See RETS specification for query options.
-#
-#    Default values query params:
-#
-#       queryType:'DMQL2',
-#       format:'COMPACT-DECODED',
-#       count:1,
-#       standardNames:0,
-#       restrictedIndicator:'***',
-#       limit:"NONE"
-###
-
-searchRets = (queryOptions) -> Promise.try () =>
-  logger.debug 'RETS method search'
-  if !queryOptions
-    throw new Error('queryOptions is required.')
-  if !queryOptions.searchType
-    throw new Error('searchType is required (ex: Property')
-  if !queryOptions.class
-    throw new Error('class is required (ex: RESI)')
-  if !queryOptions.query
-    throw new Error('query is required (ex: (MatrixModifiedDT=2014-01-01T00:00:00.000+) )')
-  finalQueryOptions = setDefaults(queryOptions, queryOptionsDefaults)
-  utils.callRetsMethod('search', @retsSession, finalQueryOptions)
-  .then (result) ->
-    result.body
-
-###
-#
-# Helper that performs a targeted RETS query and parses results.
-#
-# @param searchType Rets resource type (ex: Property)
-# @param classType Rets class type (ex: RESI)
-# @param query Rets query string. See RETS specification - (ex: MatrixModifiedDT=2014-01-01T00:00:00.000+)
-# @param _options Search query options (optional).
-#    See RETS specification for query options.
-#
-#    Default values query params:
-#
-#       queryType:'DMQL2',
-#       format:'COMPACT-DECODED',
-#       count:1,
-#       standardNames:0,
-#       restrictedIndicator:'***',
-#       limit:"NONE"
-#
-#       Please note that queryType and format are immutable.
-###
-
-query = (resourceType, classType, queryString, options) -> Promise.try () =>
-  baseOpts =
-    searchType: resourceType
-    class: classType
-    query: queryString
-  finalQueryOptions = setDefaults(baseOpts, options)
-  # make sure queryType and format will use the searchRets defaults
-  delete finalQueryOptions.queryType
-  delete finalQueryOptions.format
-  @searchRets(finalQueryOptions)
-  .then utils.parseCompact
-
-
-module.exports = (_retsSession) ->
-  if !_retsSession
-    throw new Error('System data not set; invoke login().')
-  retsSession: Promise.promisify(_retsSession)
-  searchRets: searchRets
-  query: query

lib/utils.coffee

-### jshint node:true ###
-### jshint -W097 ###
-'use strict'
-
-urlUtil = require('url')
-Promise = require('bluebird')
-logger = require('winston')
-xmlParser = Promise.promisify(require('xml2js').parseString)
-
-replycodes = require('./replycodes')
-
-
-class RetsReplyError extends Error
-  constructor: (@replyCode, @replyText) ->
-    @name = 'RetsReplyError'
-    @replyTag = if replycodes.tagMap[@replyCode]? then replycodes.tagMap[@replyCode] else 'unknown reply code'
-    @message = "RETS Server replied with an error code - ReplyCode #{@replyCode} (#{@replyTag}); ReplyText: #{@replyText}"
-    Error.captureStackTrace(this, RetsReplyError)
-
-
-class RetsServerError extends Error
-  constructor: (@retsMethod, @httpStatus) ->
-    @name = 'RetsServerError'
-    @message = "Error while attempting #{@retsMethod} - HTTP Status #{@httpStatus} returned"
-    Error.captureStackTrace(this, RetsServerError)
-
-
-replyCodeCheck = (result) -> Promise.try () ->
-  # I suspect we'll want to allow 20208 replies through as well, but I'll wait to handle that until I can see
-  # it in action myself or get info (or a PR) from someone else who can 
-  if result.RETS.$.ReplyCode == '0'
-    return result
-  throw new RetsReplyError(result.RETS.$.ReplyCode, result.RETS.$.ReplyText)
-
-  
-hex2a = (hexx) ->
-  if !hexx?
-    return null
-  hex = hexx.toString()
-  # force conversion
-  str = ''
-  i = 0
-  while i < hex.length
-    str += String.fromCharCode(parseInt(hex.substr(i, 2), 16))
-    i += 2
-  str
-
-
-# Returns a valid url for use with RETS server. If target url just contains a path, fullURL's protocol and host will be utilized.
-getValidUrl = (targetUrl, fullUrl) ->
-  loginUrlObj = urlUtil.parse(fullUrl, true, true)
-  targetUrlObj = urlUtil.parse(targetUrl, true, true)
-  if targetUrlObj.host == loginUrlObj.host
-    return targetUrl
-  fixedUrlObj =
-    protocol: loginUrlObj.protocol
-    slashes: true
-    host: loginUrlObj.host
-    pathname: targetUrlObj.pathname
-    query: targetUrlObj.query
-  urlUtil.format fixedUrlObj
-
-
-callRetsMethod = (methodName, retsSession, queryOptions) ->
-  Promise.try () ->
-    retsSession(qs: queryOptions)
-  .catch (error) ->
-    logger.debug "RETS #{methodName} error:\n" + JSON.stringify(error)
-    Promise.reject(error)
-  .spread (response, body) ->
-    if response.statusCode != 200
-      error = new RetsServerError(methodName, response.statusCode)
-      logger.debug "RETS #{methodName} error:\n" + error.message
-      return Promise.reject(error)
-    response: response
-    body: body
-
-
-croppedSlice = (data, delimiter) ->
-  crop = delimiter.length
-  data.slice(crop, -crop).split(delimiter)
-
-
-parseCompact = (rawXml, subtag) -> Promise.try () ->
-  xmlParser(rawXml)
-  .then replyCodeCheck
-  .then (parsedXml) ->
-    result = {}
-    if subtag
-      resultBase = parsedXml.RETS[subtag]?[0]
-      if !resultBase
-        throw new Error("Failed to parse #{subtag} XML: #{resultBase}")
-      delimiter = '\t'
-      result.info = resultBase.$
-      result.type = subtag
-    else
-      resultBase = parsedXml.RETS
-      delimiter = hex2a(parsedXml.RETS.DELIMITER?[0]?.$?.value)
-      if !delimiter
-        throw new Error('No specified delimiter.')
-      result.count = parsedXml.RETS.COUNT?[0]?.$?.Records
-      result.maxRowsExceeded = parsedXml.RETS.MAXROWS?
-    
-    rawColumns = resultBase.COLUMNS?[0]
-    if !rawColumns
-      throw new Error("Failed to parse columns XML: #{resultBase.COLUMNS}")
-    rawData = resultBase.DATA
-    if !rawData?.length
-      throw new Error("Failed to parse data XML: #{rawData}")
-    
-    columns = croppedSlice(rawColumns, delimiter)
-    results = []
-    for row in rawData
-      data = croppedSlice(row, delimiter)
-      model = {}
-      for column,i in columns
-        model[column] = data[i]
-      results.push model
-    result.results = results
-    result.replyCode = parsedXml.RETS.$.ReplyCode
-    result.replyTag = replycodes.tagMap[parsedXml.RETS.$.ReplyCode]
-    result.replyText = parsedXml.RETS.$.ReplyText
-    result
-
-
-module.exports = 
-  replyCodeCheck: replyCodeCheck
-  hex2a: hex2a
-  getValidUrl: getValidUrl
-  croppedSlice: croppedSlice
-  callRetsMethod: callRetsMethod
-  parseCompact: parseCompact
-  RetsReplyError: RetsReplyError
-  RetsServerError: RetsServerError

New alerts

Trivial Package

Supply chain risk

Packages less than 10 lines of code are easily copied into your own project and may not warrant the additional supply chain risk of an external dependency.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

48451

39.18%

Number of package files

21

61.54%

Number of lines in readme file

209

42.18%

Worsened metrics

Dependency count

10

11.11%

Lines of code

6

-80.65%

Number of medium supply chain risk alerts

1

Infinity%

Dependency changes

• + Added

  node-expat@^2.3.10

• + Added

  through2@^2.0.0

• + Added

  bindings@1.5.0
  (transitive)

• + Added

  core-util-is@1.0.3
  (transitive)

• + Added

  file-uri-to-path@1.0.0
  (transitive)

• + Added

  inherits@2.0.4
  (transitive)

• + Added

  isarray@1.0.0
  (transitive)

• + Added

  nan@2.26.2
  (transitive)

• + Added

  node-expat@2.4.1
  (transitive)

• + Added

  process-nextick-args@2.0.1
  (transitive)

• + Added

  readable-stream@2.3.8
  (transitive)

• + Added

  safe-buffer@5.1.2
  (transitive)

• + Added

  string_decoder@1.1.1
  (transitive)

• + Added

  through2@2.0.5
  (transitive)

• + Added

  util-deprecate@1.0.2
  (transitive)

• + Added

  xtend@4.0.2
  (transitive)

• - Removed

  xml2js@~0.4.4

• - Removed

  sax@1.6.0
  (transitive)

• - Removed

  xml2js@0.4.23
  (transitive)

• - Removed

  xmlbuilder@11.0.1
  (transitive)