upgrade lodash to fix vunerability

add supported for listening on interested nodes

.npmignore

@@ -1,3 +1,3 @@
 # Dependency directories
-node_modules
-test
+test
+.travis.yml

README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://travis-ci.org/Sai1919/xml-streamer.svg?branch=master)](https://travis-ci.org/Sai1919/xml-streamer)
 ## Motivation
 
-You use [Node.js](https://nodejs.org) for speed? You process XML streams? Then you want the fastest XML to JS parser: `xml-streamer`, based on [node-expat](https://github.com/astro/node-expat)
+You use [Node.js](https://nodejs.org) for speed? You process XML streams? Then you want the fastest XML to JS parser: `xml-streamer`, based on [node-expat](https://github.com/astro/node-expat) and It implements the Node.js `stream.Transform API`.
 
 ## Install
 
@@ -13,12 +13,40 @@ npm install xml-streamer
 
 ## Basic Usage
 
-`xml-streamer can be used in three
- ways`
+`xml-streamer can be used in four ways`
 
 ```javascript
+// 1. By passing the resourcePath and reading data by calling `read` method instead listening for data events.
 
-// 1. By listening for interested nodes.
+(function () {
+  "use strict";
+
+  var Parser = require('xml-streamer')
+  
+  var opts = {resourcePath: '/items/item'}
+
+  var parser = new Parser(opts)
+  
+  parser.on('end', function () {
+    // parsing ended no more data events will be raised
+  })
+
+  parser.on('error', function (error) {
+    // error occurred
+    // NOTE: when error event emitted no end event will be emitted
+    console.error(error)
+  })
+
+  xmlStream.pipe(parser) // pipe your input xmlStream to parser.
+  // readable
+  parser.on('readable', function () {
+     // if you don't want to consume "data" on "data" events you can wait for readable event and consume data by calling parser.read() 
+  })
+  // after readable event occured you can call read method and get data.
+  parser.read() // will return one object at a time.
+}())
+
+// 2. By listening for interested nodes.
 
 (function () {
   "use strict";
@@ -50,7 +78,7 @@ npm install xml-streamer
   })
 }())
 
-// 2. By passing a resource path.
+// 3. By passing a resource path.
 
 (function () {
   "use strict";
@@ -81,31 +109,19 @@ npm install xml-streamer
   })
 }())
 
-// 2. By passing the resourcePath as shown in 2 method and reading data by calling `read` method instead listening for data events.
+// 4. By passing a string or buffer to parse function
 
 (function () {
   "use strict";
 
   var Parser = require('xml-streamer')
   
-  var opts = {resourcePath: '/items/item'}
+  var opts = {resourcePath: '/items/item'} // resourcePath is manditory when using parse method
 
   var parser = new Parser(opts)
   
-  parser.on('end', function () {
-    // parsing ended no more data events will be raised
-  })
-
-  parser.on('error', function (error) {
-    // error occurred
-    // NOTE: when error event emitted no end event will be emitted
-    console.error(error)
-  })
-
-  xmlStream.pipe(parser) // pipe your input xmlStream to parser.
-  // readable
-  parser.on('readable', function () {
-      // if you don't want to consume "data" on "data" events you can wait for readable event and consume data by calling parser.read() 
+  parser.parse(stringOrBuffer, function (err, data) {
+    // consume data here
   })
 }())
 
@@ -121,6 +137,7 @@ npm install xml-streamer
 * `#resume()` resumes
 * `#read()` returns object if stream is readable
 
+
 ## Available Constructor Options
 
 * `resourcePath`: `Type: String` Optional field. Used to extract the XML nodes that you are interested in. 
@@ -144,6 +161,8 @@ npm install xml-streamer
            if you are interested in `subitem` nodes then resourcePath would be: `/items/item/subitem`
            if you are interested in `items` nodes then resourcePath would be: `/items`
 
+
+
 * `emitOnNodeName`: `Type: Boolean` Optional field. Set this to true if you want to listen on node names instead of data event. `default: false`
             
             // Ex: consider the above XML snippet
@@ -162,7 +181,9 @@ npm install xml-streamer
              ```
 
                 `NOTE:` when you set `emitOnNodeName:true` "data" events are emitted normally. So make sure you don't listen for both the events.
-  
+
+
+
 * `attrsKey`: `Type: String` Optional field. pass the value with which you want to reference attributes of a node in its object form. `default: '$'`
                   
 * `textKey`: `Type: String` Optional field. pass the value with which you want to reference node value in its object form. `default: '_'`
@@ -186,10 +207,33 @@ npm install xml-streamer
                      // Then set `attrsKey= "attrs"` and `textKey= "text"`
 
 
+* `explicitArray`: `Type: Boolean` Optional field. `Default value is true`. All children nodes will come in an array when this option is true.
+
+            // Ex: For example let the XML be
+                ```xml
+                    <?xml version="1.0" encoding="utf-8"?>
+                    <items>
+                        <item id="1" test= 'hello'>
+                            <subitem sub= "2">two</subitem>
+                        </item>
+                    </items>
+                ```
+            // if explicitArray is true and resourcePath is /items/item. 
+            // Output for above xml will be
+                ```javascript
+                        [
+                           { '$': { id: '1', test: 'hello' },
+                             subitem: { '$': { sub: '2' }, _: 'two' } },
+                        ]
+                ```
+        `caution:` When explicitArray set to false and if there are multiple children nodes with same name then last node will override all preceding nodes. 
+
+
 ## upcoming features
 
 1. `handling of compressed streams`
 2. `handling of different encodings`
+3. `Filtering of objects extracted from resourcePath based on xpaths and json paths`
 
 
 ## Namespace handling

package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.1",
+  "version": "0.2.1",
   "name": "xml-streamer",
   "description": "XML stream parser for parsing large files efficiently with less usage of memory.",
   "author": {
@@ -14,7 +14,9 @@
     "xml parser",
     "xml parsing",
     "xml2js",
-    "xmltojs"
+    "xmltojs",
+    "node-expat",
+    "expat"
   ],
   "license": "MIT",
   "repository": {
@@ -23,7 +25,7 @@
   },
   "dependencies": {
     "node-expat": "2.3.15",
-    "lodash": "4.16.6"
+    "lodash": "4.17.5"
   },
   "devDependencies": {
     "mocha": "^1.21.4",
@@ -42,6 +44,9 @@
     }
   ],
   "standard": {
-    "globals": [ "describe", "it" ]
+    "globals": [
+      "describe",
+      "it"
+    ]
   }
 }

parser.js

@@ -8,8 +8,10 @@ var defaults = {
   resourcePath: '',
   emitOnNodeName: false,
   attrsKey: '$',
-  textKey: '_'
+  textKey: '_',
+  explicitArray: true
 }
+
 function XmlParser (opts) {
   this.opts = _.defaults(opts, defaults)
   this.parserState = new ParserState()
@@ -31,13 +33,53 @@ XmlParser.prototype.checkForInterestedNodeListeners = function () {
 
 XmlParser.prototype._transform = function (chunk, encoding, callback) {
   if (encoding !== 'buffer') this.emit('error', new Error('unsupported encoding'))
-  if (this.parserState.isRootNode) this.checkForInterestedNodeListeners()
 
-  this.parse(chunk)
+  this.processChunk(chunk)
   callback()
 }
 
-XmlParser.prototype.parse = function (chunk) {
+XmlParser.prototype.processChunk = function (chunk) {
+  var parser = this.parser
+  var state = this.parserState
+
+  if (state.isRootNode) {
+    this.checkForInterestedNodeListeners()
+    registerEvents.call(this)
+  }
+
+  if (typeof chunk === 'string') {
+    if (!parser.parse('', true)) processError.call(this)
+  } else {
+    if (!parser.parse(chunk.toString())) processError.call(this)
+  }
+}
+
+XmlParser.prototype.parse = function (chunk, cb) {
+  var parser = this.parser
+  var state = this.parserState
+  var error
+
+  if (state.isRootNode) {
+    this.checkForInterestedNodeListeners()
+    registerEvents.call(this)
+  }
+
+  if (typeof chunk === Buffer) chunk = chunk.toString()
+
+  this.on('error', function (err) {
+    error = err
+  })
+
+  if (!parser.parse(chunk)) {
+    error = processError.call(this)
+  }
+
+  if (error) return cb(error)
+
+  return cb(null, this._readableState.buffer)
+}
+
+function registerEvents () {
   var scope = this
   var parser = this.parser
   var state = this.parserState
@@ -46,50 +88,30 @@ XmlParser.prototype.parse = function (chunk) {
   var attrsKey = this.opts.attrsKey
   var textKey = this.opts.textKey
   var interestedNodes = state.interestedNodes
+  var explicitArray = this.opts.explicitArray
 
-  if (state.isRootNode) registerEvents()
+  parser.on('startElement', function (name, attrs) {
+    if (state.isRootNode) validateResourcePath(name)
+    state.currentPath = state.currentPath + '/' + name
+    checkForResourcePath(name)
+    if (state.isPathfound) processStartElement(name, attrs)
+  })
 
-  if (typeof chunk === 'string') {
-    if (!parser.parse('', true)) processError()
-  } else {
-    if (!parser.parse(chunk.toString())) processError()
-  }
+  parser.on('endElement', function (name) {
+    state.lastEndedNode = name
+    lastIndex = state.currentPath.lastIndexOf('/' + name)
+    state.currentPath = state.currentPath.substring(0, lastIndex)
+    if (state.isPathfound) processEndElement(name)
+    checkForResourcePath(name)
+  })
 
-  function registerEvents () {
-    parser.on('startElement', function (name, attrs) {
-      if (state.isRootNode) validateResourcePath(name)
-      state.currentPath = state.currentPath + '/' + name
-      checkForResourcePath(name)
-      if (state.isPathfound) processStartElement(name, attrs)
-    })
+  parser.on('text', function (text) {
+    if (state.isPathfound) processText(text)
+  })
 
-    parser.on('endElement', function (name) {
-      state.lastEndedNode = name
-      lastIndex = state.currentPath.lastIndexOf('/' + name)
-      state.currentPath = state.currentPath.substring(0, lastIndex)
-      if (state.isPathfound) processEndElement(name)
-      checkForResourcePath(name)
-    })
-
-    parser.on('text', function (text) {
-      if (state.isPathfound) processText(text)
-    })
-
-    parser.on('error', function (err) {
-      processError(err)
-    })
-  }
-
-  function processError (err) {
-    var error = ''
-
-    if (err) {
-      error = err
-    } else {
-      error = parser.getError()
-    }
-    scope.emit('error', new Error(error + ' at line no: ' + parser.getCurrentLineNumber()))
-  }
+  parser.on('error', function (err) {
+    processError.call(this, err)
+  })
 
   function processStartElement (name, attrs) {
     if (!name) return
@@ -105,15 +127,20 @@ XmlParser.prototype.parse = function (chunk) {
     var tokens = path.split('.')
 
     for (var i = 0; i < tokens.length; i++) {
-      if (tempObj[tokens[i]]) {
+      if (tempObj[tokens[i]] && !(explicitArray === false && i === tokens.length - 1)) {
         tempObj = tempObj[tokens[i]]
       } else {
-        tempObj[tokens[i]] = []
+        // if explicitArray is true then create each node as array
+        // irrespective of how many nodes are there with same name.
+        tempObj[tokens[i]] = explicitArray ? [] : obj
         tempObj = tempObj[tokens[i]]
       }
       if (Array.isArray(tempObj) && i !== tokens.length - 1) tempObj = tempObj[tempObj.length - 1]
     }
-    tempObj.push(obj)
+
+    if (Array.isArray(tempObj)) {
+      tempObj.push(obj)
+    }
   }
 
   function processEndElement (name) {
@@ -173,14 +200,20 @@ XmlParser.prototype.parse = function (chunk) {
       if (tempObj[tokens[i]]) {
         tempObj = tempObj[tokens[i]]
       } else {
-        tempObj[tokens[i]] = []
+        tempObj[tokens[i]] = explicitArray ? [] : {}
         tempObj = tempObj[tokens[i]]
       }
       if (Array.isArray(tempObj) && i !== tokens.length - 1) tempObj = tempObj[tempObj.length - 1]
     }
-    var obj = tempObj[tempObj.length - 1]
-    if (!obj[textKey]) obj[textKey] = ''
-    obj[textKey] = obj[textKey] + text
+
+    if (Array.isArray(tempObj)) {
+      var obj = tempObj[tempObj.length - 1]
+      if (!obj[textKey]) obj[textKey] = ''
+      obj[textKey] = obj[textKey] + text
+    } else {
+      if (!tempObj[textKey]) tempObj[textKey] = ''
+      tempObj[textKey] = tempObj[textKey] + text
+    }
   }
 
   function checkForResourcePath (name) {
@@ -243,8 +276,22 @@ XmlParser.prototype.parse = function (chunk) {
   }
 }
 
+function processError (err) {
+  var parser = this.parser
+  var error = ''
+
+  if (err) {
+    error = err
+  } else {
+    error = parser.getError()
+  }
+  error = new Error(error + ' at line no: ' + parser.getCurrentLineNumber())
+  this.emit('error', error)
+  return error
+}
+
 XmlParser.prototype._flush = function (callback) {
-  this.parse('', true)
+  this.processChunk('')
   callback()
 }
 

test/test.js

@@ -1227,4 +1227,319 @@ describe('Tests', function () {
       xmlStream.pipe(parser)
     })
   })
+
+  describe('Parse funtion should work properly', function () {
+    it('should properly parse a simple file.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/item.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item'})
+      var expectedData = [
+                           { '$': { id: '1', test: 'hello' },
+                             subitem:
+                             [ { '$': { sub: 'TESTING SUB' }, _: 'one' },
+                                  { '$': { sub: '2' }, _: 'two' } ] },
+                              { '$': { id: '2' },
+                                subitem: [ { _: 'three' }, { _: 'four' }, { _: 'five' } ] } ]
+
+      parser.parse(xml.toString(), function (err, data) {
+        if (err) done(err)
+        data.should.deepEqual(expectedData)
+        done()
+      })
+    })
+
+    it('should properly parse a medium size file.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/medium.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item'})
+
+      parser.parse(xml, function (err, data) {
+        if (err) done(err)
+        data.length.should.equal(10)
+        done()
+      })
+    })
+
+    it('should properly parse a file containing many nodes.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/manyItems.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item'})
+
+      parser.parse(xml, function (err, data) {
+        if (err) done(err)
+        data.length.should.equal(296)
+        done()
+      })
+    })
+
+    it('should properly parse a xml simple file in which nodes contain text values randomly.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/randomText.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item'})
+      var expectedData = [ { '$': { 'id': '1', 'test': 'hello' }, '_': ' item  one  two',
+                             'subitem': [ { '$': { 'sub': 'TESTING SUB' }, '_': 'one' },
+                                          { '$': { 'sub': '2' }, '_': 'two' } ] },
+                           { '$': { 'id': '2' }, '_': ' item  one two three  four',
+                             'subitem': [ { '_': 'three' }, { '_': 'four' }, { '_': 'five' } ] }
+                         ]
+
+      parser.parse(xml, function (err, data) {
+        if (err) done(err)
+        data.should.deepEqual(expectedData)
+        data.length.should.equal(2)
+        done()
+      })
+    })
+
+    it('should properly parse a huge file.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/hugeFile.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item'})
+      // console.log(parser)
+      parser.parse(xml, function (err, data) {
+        if (err) done(err)
+        data.length.should.equal(2072)
+        done()
+      })
+    })
+
+    it('should properly return error if the xml file is corrupted.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/corrupted.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item'})
+
+      parser.parse(xml, function (err, data) {
+        // console.log(err)
+        err.message.should.equal('mismatched tag at line no: 11')
+        should(data).not.be.ok()
+        done()
+      })
+    })
+  })
+
+  describe('should respect explicitArray constructor option', function () {
+    it('should properly parse a simple file with explicitArray set to false.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/item.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item', explicitArray: false})
+      var expectedData = [
+                           { '$': { id: '1', test: 'hello' },
+                             subitem: { '$': { sub: '2' }, _: 'two' } },
+                              { '$': { id: '2' },
+                                subitem: { _: 'five' } } ]
+
+      parser.parse(xml.toString(), function (err, data) {
+        if (err) done(err)
+        // console.log('data=', JSON.stringify(data))
+        data.should.deepEqual(expectedData)
+        done()
+      })
+    })
+
+    it('should properly parse a medium size file with explicitArray set to false.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/medium.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item', explicitArray: false})
+      var expectedData = [
+         {
+            "$":{
+               "id":"1",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         },
+         {
+            "$":{
+               "id":"2"
+            },
+            "subitem":{
+               "_":"five"
+            }
+         },
+         {
+            "$":{
+               "id":"3",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         },
+         {
+            "$":{
+               "id":"4",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         },
+         {
+            "$":{
+               "id":"5",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         },
+         {
+            "$":{
+               "id":"6",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         },
+         {
+            "$":{
+               "id":"7",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         },
+         {
+            "$":{
+               "id":"8",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         },
+         {
+            "$":{
+               "id":"9",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         },
+         {
+            "$":{
+               "id":"10",
+               "test":"hello"
+            },
+            "subitem":{
+               "$":{
+                  "sub":"2"
+               },
+               "_":"two"
+            }
+         }
+      ]
+      parser.parse(xml, function (err, data) {
+        if (err) done(err)
+
+        data.should.deepEqual(expectedData)
+        data.length.should.equal(10)
+        done()
+      })
+    })
+
+    it('should properly parse a file containing many nodes when explicitArray set to false.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/manyItems.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item', explicitArray: false})
+
+      parser.parse(xml, function (err, data) {
+        if (err) done(err)
+
+        data.length.should.equal(296)
+        done()
+      })
+    })
+
+    it('should properly parse a xml simple file in which nodes contain text values randomly when explicitArray set to false.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/randomText.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item', explicitArray: false})
+      var expectedData = [ { '$': { 'id': '1', 'test': 'hello' }, '_': ' item  one  two',
+                             'subitem': { '$': { 'sub': '2' }, '_': 'two' } },
+                           { '$': { 'id': '2' }, '_': ' item  one two three  four',
+                             'subitem': { '_': 'five' } }
+                         ]
+
+      parser.parse(xml, function (err, data) {
+        if (err) done(err)
+
+        data.should.deepEqual(expectedData)
+        data.length.should.equal(2)
+        done()
+      })
+    })
+
+    it('should properly parse a huge file with explicitArray set to false.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/hugeFile.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item', explicitArray: false})
+      // console.log(parser)
+      parser.parse(xml, function (err, data) {
+        if (err) done(err)
+        data.length.should.equal(2072)
+        done()
+      })
+    })
+
+    it('should properly return error if the xml file is corrupted.', function (done) {
+      var xml = fs.readFileSync('./test/TestFiles/corrupted.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item', explicitArray: false})
+
+      parser.parse(xml, function (err, data) {
+        // console.log(err)
+        err.message.should.equal('mismatched tag at line no: 11')
+        should(data).not.be.ok()
+        done()
+      })
+    })
+
+    it('should properly generate objects when special symbols are passed as attrs and text keys and explicitArray is false in the options.', function (done) {
+      var xmlStream = fs.createReadStream('./test/TestFiles/item.xml')
+      var parser = new ParserFactory({resourcePath: '/items/item', attrsKey: '!', textKey: '%', explicitArray: false})
+      var expectedData = [
+                           { '!': { id: '1', test: 'hello' },
+                             subitem: { '!': { sub: '2' }, '%': 'two' } },
+                           { '!': { id: '2' },
+                                subitem: { '%': 'five' } } ]
+      var actualData = []
+      var dataEventCount = 0
+
+      parser.on('data', function (data) {
+        actualData.push(data)
+        dataEventCount++
+      })
+
+      parser.on('error', function (err) {
+        done(err)
+      })
+
+      parser.on('end', function () {
+        // console.log('actualData=', JSON.stringify(actualData, null, 1))
+        // console.log('dataEventCount=', dataEventCount)
+        actualData.should.deepEqual(expectedData)
+        dataEventCount.should.equal(2)
+        done()
+      })
+      xmlStream.pipe(parser)
+    })
+  })
 })