[Token] Correctly split quoted components

Stencil/Tokenizer.swift

@@ -1,41 +1,68 @@
 import Foundation
 
 
+/// Split a string by spaces leaving quoted phrases together
+func smartSplit(value: String) -> [String] {
+  var word = ""
+  var separator: Character = " "
+  var components: [String] = []
+
+  for character in value.characters {
+    if character == separator {
+      if separator != " " {
+        word.append(separator)
+      }
+
+      if !word.isEmpty {
+        components.append(word)
+        word = ""
+      }
+
+      separator = " "
+    } else {
+      if separator == " " && (character == "'" || character == "\"") {
+        separator = character
+      }
+      word.append(character)
+    }
+  }
+
+  if !word.isEmpty {
+    components.append(word)
+  }
+
+  return components
+}
+
+
 public enum Token : Equatable {
   /// A token representing a piece of text.
-  case Text(value:String)
+  case Text(value: String)
 
   /// A token representing a variable.
-  case Variable(value:String)
+  case Variable(value: String)
 
   /// A token representing a comment.
-  case Comment(value:String)
+  case Comment(value: String)
 
   /// A token representing a template block.
-  case Block(value:String)
+  case Block(value: String)
 
   /// Returns the underlying value as an array seperated by spaces
   public func components() -> [String] {
-    // TODO: Make this smarter and treat quoted strings as a single component
-    let characterSet = NSCharacterSet.whitespaceAndNewlineCharacterSet()
-
-    func strip(value: String) -> [String] {
-      return value.stringByTrimmingCharactersInSet(characterSet).componentsSeparatedByCharactersInSet(characterSet)
-    }
-
     switch self {
     case .Block(let value):
-      return strip(value)
+      return smartSplit(value)
     case .Variable(let value):
-      return strip(value)
+      return smartSplit(value)
     case .Text(let value):
-      return strip(value)
+      return smartSplit(value)
     case .Comment(let value):
-      return strip(value)
+      return smartSplit(value)
     }
   }
 
-  public var contents:String {
+  public var contents: String {
     switch self {
     case .Block(let value):
       return value
@@ -49,7 +76,8 @@ public enum Token : Equatable {
   }
 }
 
-public func ==(lhs:Token, rhs:Token) -> Bool {
+
+public func == (lhs: Token, rhs: Token) -> Bool {
   switch (lhs, rhs) {
   case (.Text(let lhsValue), .Text(let rhsValue)):
     return lhsValue == rhsValue

StencilSpecs/TokenSpec.swift

@@ -0,0 +1,32 @@
+import Spectre
+import Stencil
+
+
+describe("Token") {
+  $0.it("can split the contents into components") {
+    let token = Token.Text(value: "hello world")
+    let components = token.components()
+
+    try expect(components.count) == 2
+    try expect(components[0]) == "hello"
+    try expect(components[1]) == "world"
+  }
+
+  $0.it("can split the contents into components with single quoted strings") {
+    let token = Token.Text(value: "hello 'kyle fuller'")
+    let components = token.components()
+
+    try expect(components.count) == 2
+    try expect(components[0]) == "hello"
+    try expect(components[1]) == "'kyle fuller'"
+  }
+
+  $0.it("can split the contents into components with double quoted strings") {
+    let token = Token.Text(value: "hello \"kyle fuller\"")
+    let components = token.components()
+
+    try expect(components.count) == 2
+    try expect(components[0]) == "hello"
+    try expect(components[1]) == "\"kyle fuller\""
+  }
+}