Add tokens and token parser

Stencil.xcodeproj/project.pbxproj

@@ -9,8 +9,11 @@
 /* Begin PBXBuildFile section */
 		7725B3CB19F92B4F002CF74B /* VariableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7725B3CA19F92B4F002CF74B /* VariableTests.swift */; };
 		7725B3CD19F92B61002CF74B /* Variable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7725B3CC19F92B61002CF74B /* Variable.swift */; };
+		7725B3CF19F94214002CF74B /* Tokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7725B3CE19F94214002CF74B /* Tokenizer.swift */; };
 		7725B3D319F9437F002CF74B /* NodeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7725B3D219F9437F002CF74B /* NodeTests.swift */; };
 		7725B3D519F9438F002CF74B /* Node.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7725B3D419F9438F002CF74B /* Node.swift */; };
+		7725B3D719F94A43002CF74B /* Parser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7725B3D619F94A43002CF74B /* Parser.swift */; };
+		7725B3D919F94A61002CF74B /* ParserTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7725B3D819F94A61002CF74B /* ParserTests.swift */; };
 		77FAAE5819F91E480029DC5E /* Stencil.h in Headers */ = {isa = PBXBuildFile; fileRef = 77FAAE5719F91E480029DC5E /* Stencil.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		77FAAE5E19F91E480029DC5E /* Stencil.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 77FAAE5219F91E480029DC5E /* Stencil.framework */; };
 		77FAAE6519F91E480029DC5E /* StencilTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77FAAE6419F91E480029DC5E /* StencilTests.swift */; };
@@ -31,8 +34,11 @@
 /* Begin PBXFileReference section */
 		7725B3CA19F92B4F002CF74B /* VariableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VariableTests.swift; sourceTree = "<group>"; };
 		7725B3CC19F92B61002CF74B /* Variable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Variable.swift; sourceTree = "<group>"; };
+		7725B3CE19F94214002CF74B /* Tokenizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Tokenizer.swift; sourceTree = "<group>"; };
 		7725B3D219F9437F002CF74B /* NodeTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NodeTests.swift; sourceTree = "<group>"; };
 		7725B3D419F9438F002CF74B /* Node.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Node.swift; sourceTree = "<group>"; };
+		7725B3D619F94A43002CF74B /* Parser.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Parser.swift; sourceTree = "<group>"; };
+		7725B3D819F94A61002CF74B /* ParserTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ParserTests.swift; sourceTree = "<group>"; };
 		77FAAE5219F91E480029DC5E /* Stencil.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Stencil.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		77FAAE5619F91E480029DC5E /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
 		77FAAE5719F91E480029DC5E /* Stencil.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Stencil.h; sourceTree = "<group>"; };
@@ -86,7 +92,9 @@
 				77FAAE5719F91E480029DC5E /* Stencil.h */,
 				77FAAE6E19F920750029DC5E /* Context.swift */,
 				7725B3CC19F92B61002CF74B /* Variable.swift */,
+				7725B3CE19F94214002CF74B /* Tokenizer.swift */,
 				7725B3D419F9438F002CF74B /* Node.swift */,
+				7725B3D619F94A43002CF74B /* Parser.swift */,
 				77FAAE5519F91E480029DC5E /* Supporting Files */,
 			);
 			path = Stencil;
@@ -107,6 +115,7 @@
 				77FAAE7019F9208C0029DC5E /* ContextTests.swift */,
 				7725B3CA19F92B4F002CF74B /* VariableTests.swift */,
 				7725B3D219F9437F002CF74B /* NodeTests.swift */,
+				7725B3D819F94A61002CF74B /* ParserTests.swift */,
 				77FAAE6219F91E480029DC5E /* Supporting Files */,
 			);
 			path = StencilTests;
@@ -228,6 +237,8 @@
 			buildActionMask = 2147483647;
 			files = (
 				77FAAE6F19F920750029DC5E /* Context.swift in Sources */,
+				7725B3CF19F94214002CF74B /* Tokenizer.swift in Sources */,
+				7725B3D719F94A43002CF74B /* Parser.swift in Sources */,
 				7725B3CD19F92B61002CF74B /* Variable.swift in Sources */,
 				7725B3D519F9438F002CF74B /* Node.swift in Sources */,
 			);
@@ -239,6 +250,7 @@
 			files = (
 				77FAAE6519F91E480029DC5E /* StencilTests.swift in Sources */,
 				7725B3D319F9437F002CF74B /* NodeTests.swift in Sources */,
+				7725B3D919F94A61002CF74B /* ParserTests.swift in Sources */,
 				7725B3CB19F92B4F002CF74B /* VariableTests.swift in Sources */,
 				77FAAE7119F9208C0029DC5E /* ContextTests.swift in Sources */,
 			);

Stencil/Node.swift

@@ -12,7 +12,7 @@ public protocol Error : Printable {
 
 }
 
-public protocol Node : Equatable {
+public protocol Node {
     func render(context:Context) -> (String?, Error?)
 }
 
@@ -28,12 +28,8 @@ public class TextNode : Node {
     }
 }
 
-public func ==(lhs:TextNode, rhs:TextNode) -> Bool {
-    return true
-}
-
 public class VariableNode : Node {
-    let variable:Variable
+    public let variable:Variable
 
     public init(variable:Variable) {
         self.variable = variable
@@ -55,7 +51,3 @@ public class VariableNode : Node {
         return (nil, nil)
     }
 }
-
-public func ==(lhs:VariableNode, rhs:VariableNode) -> Bool {
-    return true
-}

Stencil/Parser.swift

@@ -0,0 +1,54 @@
+//
+//  Parser.swift
+//  Stencil
+//
+//  Created by Kyle Fuller on 23/10/2014.
+//  Copyright (c) 2014 Cocode. All rights reserved.
+//
+
+import Foundation
+
+public class TokenParser {
+    private var tokens:[Token]
+
+    public init(tokens:[Token]) {
+        self.tokens = tokens
+    }
+
+    public func parse() -> [Node] {
+        return parse(nil)
+    }
+
+    public func parse(parse_until:((parser:TokenParser, token:Token) -> (Bool))?) -> [Node] {
+        var nodes = [Node]()
+
+        while tokens.count > 0 {
+            let token = nextToken()!
+
+            switch token {
+            case .Text(let text):
+                nodes.append(TextNode(text: text))
+            case .Variable(let variable):
+                nodes.append(VariableNode(variable: variable))
+            case .Block(let value):
+                continue
+            case .Comment(let value):
+                continue
+            }
+        }
+
+        return nodes
+    }
+
+    public func nextToken() -> Token? {
+        if tokens.count > 0 {
+            return tokens.removeAtIndex(0)
+        }
+
+        return nil
+    }
+
+    public func prependToken(token:Token) {
+        tokens.insert(token, atIndex: 0)
+    }
+}

Stencil/Tokenizer.swift

@@ -0,0 +1,48 @@
+//
+//  Tokenizer.swift
+//  Stencil
+//
+//  Created by Kyle Fuller on 23/10/2014.
+//  Copyright (c) 2014 Cocode. All rights reserved.
+//
+
+import Foundation
+
+public enum Token : Equatable {
+    case Text(value:String)
+    case Variable(value:String)
+    case Comment(value:String)
+    case Block(value:String)
+
+    /// Returns the underlying value as an array seperated by spaces
+    func components() -> [String] {
+        // TODO: Make this smarter and treat quoted strings as a single component
+        let characterSet = NSCharacterSet.whitespaceAndNewlineCharacterSet()
+
+        switch self {
+            case .Block(let value):
+                return value.value.stringByTrimmingCharactersInSet(characterSet).componentsSeparatedByCharactersInSet(characterSet)
+            case .Variable(let value):
+                return value.value.stringByTrimmingCharactersInSet(characterSet).componentsSeparatedByCharactersInSet(characterSet)
+            case .Text(let value):
+                return value.value.stringByTrimmingCharactersInSet(characterSet).componentsSeparatedByCharactersInSet(characterSet)
+            case .Comment(let value):
+                return value.value.stringByTrimmingCharactersInSet(characterSet).componentsSeparatedByCharactersInSet(characterSet)
+        }
+    }
+}
+
+public func ==(lhs:Token, rhs:Token) -> Bool {
+    switch (lhs, rhs) {
+        case (.Text(let lhsValue), .Text(let rhsValue)):
+            return lhsValue == rhsValue
+        case (.Variable(let lhsValue), .Variable(let rhsValue)):
+            return lhsValue == rhsValue
+        case (.Block(let lhsValue), .Block(let rhsValue)):
+            return lhsValue == rhsValue
+        case (.Comment(let lhsValue), .Comment(let rhsValue)):
+            return lhsValue == rhsValue
+        default:
+            return false
+    }
+}

Stencil/Variable.swift

@@ -8,7 +8,7 @@
 
 import Foundation
 
-public struct Variable {
+public struct Variable : Equatable {
     public let variable:String
 
     public init(_ variable:String) {
@@ -49,3 +49,7 @@ public struct Variable {
         return current
     }
 }
+
+public func ==(lhs:Variable, rhs:Variable) -> Bool {
+    return lhs.variable == rhs.variable
+}

StencilTests/NodeTests.swift

@@ -28,12 +28,6 @@ class TextNodeTests: NodeTests {
 
         XCTAssertEqual(result.0!, "Hello World")
     }
-
-    func testTwoIdenticalTextNodesAreEqual() {
-        let node1 = TextNode(text:"Hello World")
-        let node2 = TextNode(text:"Hello World")
-        XCTAssertEqual(node1, node2)
-    }
 }
 
 class VariableNodeTests: NodeTests {
@@ -50,10 +44,4 @@ class VariableNodeTests: NodeTests {
 
         XCTAssertEqual(result.0!, "27")
     }
-
-    func testTwoIdenticalVariableNodesAreEqual() {
-        let node1 = VariableNode(variable:Variable("name"))
-        let node2 = VariableNode(variable:Variable("name"))
-        XCTAssertEqual(node1, node2)
-    }
 }

StencilTests/ParserTests.swift

@@ -0,0 +1,48 @@
+//
+//  ParserTests.swift
+//  Stencil
+//
+//  Created by Kyle Fuller on 23/10/2014.
+//  Copyright (c) 2014 Cocode. All rights reserved.
+//
+
+import Cocoa
+import XCTest
+import Stencil
+
+class TokenParserTests: XCTestCase {
+    func testParsingTextToken() {
+        let parser = TokenParser(tokens: [
+            Token.Text(value: "Hello World")
+        ])
+
+        let nodes = parser.parse()
+        let node = nodes.first as TextNode!
+
+        XCTAssertEqual(nodes.count, 1)
+        XCTAssertEqual(node.text, "Hello World")
+    }
+
+    func testParsingVariableToken() {
+        let parser = TokenParser(tokens: [
+            Token.Variable(value: "name")
+        ])
+
+        let nodes = parser.parse()
+        let node = nodes.first as VariableNode!
+        let variable = node.variable
+
+        XCTAssertEqual(nodes.count, 1)
+        XCTAssertEqual(variable, Variable("name"))
+    }
+
+    func testParsingCommentToken() {
+        let parser = TokenParser(tokens: [
+            Token.Comment(value: "Secret stuff!")
+        ])
+
+        let nodes = parser.parse()
+
+        XCTAssertEqual(nodes.count, 0)
+    }
+}