tabshift/xml-streamer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup node-expat

Browse Source
This commit is contained in:
Dror Gluska
2019-05-31 22:57:03 +03:00
parent 46885d9ede
commit dec131431d
8 changed files with 11 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.vscode/settings.json vendored
View File

@@ -3,6 +3,7 @@
"Dror",
"Gluska",
"Teja",
"drorgl",
"gmail",
"saitejas",
"xmltojs"

3
package.json
View File

@@ -24,8 +24,7 @@
"url": "https://github.com/Sai1919/xml-streamer"
},
"dependencies": {
"lodash": "4.17.11",
"node-expat": "2.3.18"
"lodash": "4.17.11"
},
"devDependencies": {
"mocha": "^6.1.4",

5
src/ltx.ts
View File

@@ -43,11 +43,6 @@ export class SaxLtx extends events.EventEmitter {
}
this.removeAllListeners();
/* Uh, yeah */
// this.write = () => {
// // nop
// };
}
public write(data: Buffer | string) {
if (typeof data !== "string") {

37
src/parser.ts
View File

@@ -1,6 +1,4 @@
/// <reference types="../typings/node-expat" />
import _ from "lodash";
import * as expat from "node-expat";
import stream from "stream";
import util from "util";
@@ -30,12 +28,12 @@ export class XmlParser extends stream.Transform {
public parserState: ParserState;
private opts: IXmlParserOptions;
private _readableState: { objectMode: true, buffer: any };
private parser: SaxLtx; // expat.Parser;
private parser: SaxLtx;
constructor(opts?: IXmlParserOptions) {
super();
this.opts = _.defaults(opts, defaults);
this.parserState = new ParserState();
this.parser = new SaxLtx(); // new expat.Parser("UTF-8");
this.parser = new SaxLtx();
this._readableState.objectMode = true;
}
@@ -67,11 +65,6 @@ export class XmlParser extends stream.Transform {
}
parser.write(chunk);
// if (typeof chunk === "string") {
// if (!parser.parse("", true)) { processError.call(this); }
// } else {
// if (!parser.parse(chunk.toString())) {processError.call(this); }
// }
}
public parse(chunk: Buffer | string, cb: (error: Error, data?: Buffer) => void) {
@@ -84,8 +77,6 @@ export class XmlParser extends stream.Transform {
registerEvents.call(this);
}
// if (chunk instanceof Buffer) { chunk = chunk.toString(); }
this.on("error", (err) => {
error = err;
});
@@ -97,9 +88,6 @@ export class XmlParser extends stream.Transform {
}
parser.write(chunk);
// if (!parser.parse(chunk)) {
// error = processError.call(this);
// }
if (error) { return cb(error); }
@@ -119,7 +107,6 @@ export class XmlParser extends stream.Transform {
function registerEvents() {
const scope = this;
// const parser: expat.Parser = this.parser;
const parser: SaxLtx = this.parser;
const state: ParserState = this.parserState;
let lastIndex;
@@ -132,7 +119,6 @@ function registerEvents() {
const preserveWhitespace = this.opts.preserveWhitespace;
parser.on("startElement", (name, attrs) => {
// console.log("start", name, attrs);
if (state.isRootNode) { state.isRootNode = false; }
state.currentPath = state.currentPath + "/" + name;
checkForResourcePath(name);
@@ -140,7 +126,6 @@ function registerEvents() {
});
parser.on("endElement", (name) => {
// console.log("end?", name, state.currentPath);
state.lastEndedNode = name;
lastIndex = state.currentPath.lastIndexOf("/" + name);
if (state.currentPath.substring(lastIndex + 1).indexOf("/") !== -1) {
@@ -148,7 +133,6 @@ function registerEvents() {
}
state.currentPath = state.currentPath.substring(0, lastIndex);
if (state.isPathfound) { processEndElement(name); }
// console.log("end!", name, state.currentPath);
checkForResourcePath(name);
});
@@ -233,7 +217,7 @@ function registerEvents() {
scope.push(tempObj);
}
function processText(text: string) {
function processText(text: string) {
if ((!text) || ((!verbatimText) && !/\S/.test(text))) {
return;
}
@@ -243,7 +227,7 @@ function registerEvents() {
if (!path) {
if (!state.object[textKey]) { state.object[textKey] = ""; }
state.object[textKey] = state.object[textKey] + text;
if ((! preserveWhitespace)) {
if ((!preserveWhitespace)) {
state.object[textKey] = state.object[textKey].replace(/\s+/g, " ").trim();
}
return;
@@ -264,7 +248,7 @@ function registerEvents() {
if (!obj[textKey]) { obj[textKey] = ""; }
obj[textKey] = obj[textKey] + text;
if ((! preserveWhitespace)) {
if ((!preserveWhitespace)) {
obj[textKey] = obj[textKey].replace(/\s+/g, " ").trim();
}
@@ -272,14 +256,14 @@ function registerEvents() {
if (!tempObj[textKey]) { tempObj[textKey] = ""; }
tempObj[textKey] = tempObj[textKey] + text;
if ((! preserveWhitespace)) {
if ((!preserveWhitespace)) {
tempObj[textKey] = tempObj[textKey].replace(/\s+/g, " ").trim();
}
}
}
function checkForResourcePath(name: string) {
function checkForResourcePath(name: string) {
if (resourcePath) {
if (state.currentPath.indexOf(resourcePath) === 0) {
state.isPathfound = true;
@@ -296,7 +280,7 @@ function registerEvents() {
}
}
function getRelativePath() {
function getRelativePath() {
let tokens;
let jsonPath;
let index;
@@ -332,8 +316,3 @@ function processError(err: Error) {
this.emit("error", error);
return error;
}
// setInterval(() => {
// console.log("handles", (process as any)._getActiveHandles());
// console.log("requests", (process as any)._getActiveRequests());
// }, 5000);

2
test/performance.spec.ts
View File

@@ -6,7 +6,7 @@ import stream from "stream";
import zlib from "zlib";
import { XmlParser } from "../src/parser";
describe("performance testing", () => {
describe.skip("performance testing", () => {
it("should properly parse more than 500 MB of file.", function(done) {
const parser = new XmlParser({ resourcePath: "/items/item" });
// var wsStream = fs.createWriteStream('./test/TestFiles/MB_and_GB_size_files/MBFile.xml')

2
tsconfig.json
View File

@@ -19,11 +19,9 @@
"lib": ["es2017"],
"typeRoots": [
"./node_modules/@types",
"./typings"
]
},
"include": [
"./typings/**/*.d.ts",
"src/**/*"
],
"exclude": [

92
typings/node-expat/index.d.ts vendored
View File

@@ -1,92 +0,0 @@
// Type definitions for node-expat 2.3.x
// Project: http://nodejs.org/
// Definitions by: Microsoft TypeScript <http://typescriptlang.org>
// Oleksandr Fedorchuk <https://github.com/kol-93>
// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
declare module "node-expat" {
import { Stream } from "stream";
import { EventEmitter } from "events";
interface NameSpace<ValueType = any> {
[key: string]: ValueType;
}
interface TypedEmitter<EventMapType extends NameSpace<any[]>> extends EventEmitter {
addListener<Event extends keyof EventMapType> (event: Event, listener: (...args: EventMapType[Event]) => void): this;
on<Event extends keyof EventMapType> (event: Event, listener: (...args: EventMapType[Event]) => void): this;
once<Event extends keyof EventMapType> (event: Event, listener: (...args: EventMapType[Event]) => void): this;
prependListener<Event extends keyof EventMapType> (event: Event, listener: (...args: EventMapType[Event]) => void): this;
prependOnceListener<Event extends keyof EventMapType> (event: Event, listener: (...args: EventMapType[Event]) => void): this;
removeListener<Event extends keyof EventMapType> (event: Event, listener: (...args: EventMapType[Event]) => void): this;
emit<Event extends keyof EventMapType> (event: Event, ...args: EventMapType[Event]): boolean;
addListener (event: string | symbol, listener: (...args: any[]) => void): this;
on (event: string | symbol, listener: (...args: any[]) => void): this;
once (event: string | symbol, listener: (...args: any[]) => void): this;
prependListener (event: string | symbol, listener: (...args: any[]) => void): this;
prependOnceListener (event: string | symbol, listener: (...args: any[]) => void): this;
removeListener (event: string | symbol, listener: (...args: any[]) => void): this;
emit (event: string | symbol, ...args: any[]): boolean;
}
interface ParserEventsMap extends NameSpace<any[]> {
startElement: [string, NameSpace<string>];
endElement: [string];
text: [string];
comment: [string];
processingInstruction: [string, string];
xmlDecl: [string | null, string | null, boolean];
startCdata: [];
endCdata: [];
entityDecl: [string | null, boolean, string | null, string | null, string | null, string | null, string | null];
end: [];
close: [];
error: [string | Error];
}
export class Parser extends Stream implements NodeJS.WritableStream, TypedEmitter<ParserEventsMap>
{
constructor(encoding:string);
readonly writable: boolean;
stop(): this;
pause(): this;
resume(): this;
parse(buf: string | Buffer, isFinal?: boolean): boolean;
setEncoding(value: string): this;
/// @TODO setUnknownEncoding
getError(): string | null;
destroy(): this;
destroySoon(): this;
write(buffer: Buffer | string, cb?: Function): boolean;
write(str: string, encoding?: string, cb?: Function): boolean;
end(cb?: Function): boolean;
end(chunk: Buffer, cb?: Function): boolean;
end(chunk: string, cb?: Function): boolean;
end(chunk: string, encding?: string, cb?: Function): boolean;
reset(this: Parser): this;
getCurrentLineNumber(this: Parser): number;
getCurrentColumnNumber(this: Parser): number;
getCurrentByteIndex(this: Parser): number;
on (event: "startElement", listener:(name:string, attrs: any)=>void):this;
on (event: "endElement", listener:(name:string)=>void):this;
on (event: "text", listener:(text:string)=>void):this;
on (event: "processingInstruction", listener:(target:any, data:any)=>void):this;
on (event: "comment", listener:(s:string)=>void):this;
on (event: "xmlDecl", listener:(version:string, encoding: string, standalone: boolean)=>void):this;
on (event: "startCdata", listener:()=>void):this;
on (event: "endCdata", listener:()=>void):this;
on (event: "entityDecl", listener:(entityName:string, isParameterEntity:boolean, value:string, base:string, systemId:string, publicId:string, notationName:string)=>void):this;
on (event: "error", listener:(e:Error)=>void):this;
on (event: "close", listener:()=>void):this;
}
export function createParser(callback?: (name: string, attributes: NameSpace<string>) => void): Parser;
}

3
typings/node-expat/project.json
View File

@@ -1,3 +0,0 @@
{
}