replaced Lexeme protocol with Token

Sources/Errors.swift

@@ -20,35 +20,16 @@ public class TemplateDoesNotExist: Error, CustomStringConvertible {
 
 public struct TemplateSyntaxError : Error, Equatable, CustomStringConvertible {
   public let reason: String
-  public private(set) var description: String
+  public var description: String { return reason }
-
+  public internal(set) var token: Token?
   public internal(set) var template: Template?
   public internal(set) var parentError: Error?
   
-  var lexeme: Lexeme? {
+  public init(reason: String, token: Token? = nil, template: Template? = nil, parentError: Error? = nil) {
-    didSet {
-      description = TemplateSyntaxError.description(reason: reason, lexeme: lexeme, template: template)
-    }
-  }
-  
-  static func description(reason: String, lexeme: Lexeme?, template: Template?) -> String {
-    guard let template = template, let lexeme = lexeme else { return reason }
-    let templateName = template.name.map({ "\($0):" }) ?? ""
-    let range = template.templateString.range(of: lexeme.contents, range: lexeme.range) ?? lexeme.range
-    let line = template.templateString.rangeLine(range)
-    let highlight = "\(String(Array(repeating: " ", count: line.offset)))^\(String(Array(repeating: "~", count: max(lexeme.contents.length - 1, 0))))"
-
-    return "\(templateName)\(line.number):\(line.offset): error: \(reason)\n"
-      + "\(line.content)\n"
-      + "\(highlight)\n"
-  }
-
-  init(reason: String, lexeme: Lexeme? = nil, template: Template? = nil, parentError: Error? = nil) {
     self.reason = reason
     self.parentError = parentError
     self.template = template
-    self.lexeme = lexeme
+    self.token = token
-    self.description = TemplateSyntaxError.description(reason: reason, lexeme: lexeme, template: template)
   }
   
   public init(_ description: String) {
@@ -95,7 +76,7 @@ open class SimpleErrorReporter: ErrorReporter {
     guard let context = context else { return error }
 
     return TemplateSyntaxError(reason: (error as? TemplateSyntaxError)?.reason ?? "\(error)",
-                               lexeme: (error as? TemplateSyntaxError)?.lexeme,
+                               token: (error as? TemplateSyntaxError)?.token,
                                template: (error as? TemplateSyntaxError)?.template ?? context.template,
                                parentError: (error as? TemplateSyntaxError)?.parentError
     )
@@ -104,7 +85,21 @@ open class SimpleErrorReporter: ErrorReporter {
   open func renderError(_ error: Error) -> String {
     guard let templateError = error as? TemplateSyntaxError else { return error.localizedDescription }
     
-    var descriptions = [templateError.description]
+    let description: String
+    if let template = templateError.template, let token = templateError.token {
+      let templateName = template.name.map({ "\($0):" }) ?? ""
+      let range = template.templateString.range(of: token.contents, range: token.range) ?? token.range
+      let line = template.templateString.rangeLine(range)
+      let highlight = "\(String(Array(repeating: " ", count: line.offset)))^\(String(Array(repeating: "~", count: max(token.contents.characters.count - 1, 0))))"
+      
+      description = "\(templateName)\(line.number):\(line.offset): error: \(templateError.reason)\n"
+        + "\(line.content)\n"
+        + "\(highlight)\n"
+    } else {
+      description = templateError.reason
+    }
+    
+    var descriptions = [description]
     
     var currentError: TemplateSyntaxError? = templateError
     while let parentError = currentError?.parentError {
@@ -127,13 +122,4 @@ extension String {
   var range: Range<String.Index> {
     return startIndex..<endIndex
   }
-  
-  var length: Int {
-    #if swift(>=3.2)
-      return count
-    #else
-      return characters.count
-    #endif
-  }
-  
 }

Sources/Inheritence.swift

@@ -161,7 +161,7 @@ class BlockNode : NodeType {
           let baseError = context.errorReporter.reportError(error)
           throw TemplateSyntaxError(
             reason: (baseError as? TemplateSyntaxError)?.reason ?? "\(baseError)",
-            lexeme: blockSuperNode.token,
+            token: blockSuperNode.token,
             template: child.template,
             parentError: baseError)
         }
@@ -178,7 +178,7 @@ class BlockNode : NodeType {
         // unless it's already set
         if var error = error as? TemplateSyntaxError {
           error.template = error.template ?? child.template
-          error.lexeme = error.lexeme ?? child.node.token
+          error.token = error.token ?? child.node.token
           throw error
         } else {
           throw error

Sources/Lexer.swift

@@ -56,7 +56,7 @@ struct Lexer {
     return tokens
   }
   
-  func lexemeLine(_ lexeme: Lexeme) -> (content: String, number: Int, offset: String.IndexDistance) {
+  func tokenLine(_ token: Token) -> (content: String, number: Int, offset: String.IndexDistance) {
     var lineNumber: Int = 0
     var offset = 0
     var lineContent = ""
@@ -64,9 +64,9 @@ struct Lexer {
     for line in templateString.components(separatedBy: CharacterSet.newlines) {
       lineNumber += 1
       lineContent = line
-      if let rangeOfLine = templateString.range(of: line), rangeOfLine.contains(lexeme.range.lowerBound) {
+      if let rangeOfLine = templateString.range(of: line), rangeOfLine.contains(token.range.lowerBound) {
         offset = templateString.distance(from: rangeOfLine.lowerBound, to:
-          lexeme.range.lowerBound)
+          token.range.lowerBound)
         break
       }
     }
@@ -76,11 +76,6 @@ struct Lexer {
   
 }
 
-protocol Lexeme {
-  var contents: String { get }
-  var range: Range<String.Index> { get }
-}
-
 class Scanner {
   let originalContent: String
   var content: String
@@ -111,7 +106,7 @@ class Scanner {
         let result = content.substring(to: index)
 
         if returnUntil {
-          range = range.lowerBound..<originalContent.index(range.upperBound, offsetBy: until.length)
+          range = range.lowerBound..<originalContent.index(range.upperBound, offsetBy: until.characters.count)
           content = substring.substring(from: until.endIndex)
           return result + until
         }

Sources/Node.swift

@@ -16,7 +16,7 @@ public func renderNodes(_ nodes:[NodeType], _ context:Context) throws -> String
       return try $0.render(context)
     } catch {
       if var error = error as? TemplateSyntaxError {
-        error.lexeme = error.lexeme ?? $0.token
+        error.token = error.token ?? $0.token
         throw error
       } else {
         throw error

Sources/Parser.swift

@@ -54,8 +54,8 @@ public class TokenParser {
             let node = try parser(self, token)
             nodes.append(node)
           } catch {
-            if var syntaxError = error as? TemplateSyntaxError, syntaxError.lexeme == nil {
+            if var syntaxError = error as? TemplateSyntaxError, syntaxError.token == nil {
-              syntaxError.lexeme = token
+              syntaxError.token = token
               throw syntaxError
             } else {
               throw error
@@ -106,11 +106,11 @@ public class TokenParser {
     do {
       return try FilterExpression(token: filterToken, parser: self)
     } catch {
-      if var syntaxError = error as? TemplateSyntaxError, syntaxError.lexeme == nil {
+      if var syntaxError = error as? TemplateSyntaxError, syntaxError.token == nil {
         if let filterTokenRange = environment.template?.templateString.range(of: filterToken, range: containingToken.range) {
-          syntaxError.lexeme = Token.block(value: filterToken, at: filterTokenRange)
+          syntaxError.token = Token.block(value: filterToken, at: filterTokenRange)
         } else {
-          syntaxError.lexeme = containingToken
+          syntaxError.token = containingToken
         }
         throw syntaxError
       } else {

Sources/Tokenizer.swift

@@ -40,7 +40,7 @@ extension String {
   }
 }
 
-public enum Token : Equatable, Lexeme {
+public enum Token : Equatable {
   /// A token representing a piece of text.
   case text(value: String, at: Range<String.Index>)
 

Tests/StencilTests/EnvironmentSpec.swift

@@ -35,8 +35,8 @@ func testEnvironment() {
     }
     
     func expectedSyntaxError(token: String, template: Template, description: String) -> TemplateSyntaxError {
-      let lexeme = Token.block(value: token, at: template.templateString.range(of: token)!)
+      let token = Token.block(value: token, at: template.templateString.range(of: token)!)
-      return TemplateSyntaxError(reason: description, lexeme: lexeme, template: template, parentError: nil)
+      return TemplateSyntaxError(reason: description, token: token, template: template, parentError: nil)
     }
     
     $0.it("reports syntax error on invalid for tag syntax") {