From a1718ae350c0afcd9dbf5d89d2ebdce7d9a6407c Mon Sep 17 00:00:00 2001
From: Andreas Ley <andreas@andreasley.ch>
Date: Sun, 12 Jan 2020 16:21:02 +0100
Subject: [PATCH] Fix for incorrect tokenization due to index difference of
 Unicode character/scalar (#286)

* Fix: `Scanner` now uses indices of the respective UnicodeScalarView

* Fix: `Scanner` now uses indices of the respective UnicodeScalarView

* Extended test for Unicode `Combining Diaeresis`

* Fixed test for combining diaeresis

* Inlined template for testing Unicode combining diaeresis

Co-authored-by: Ilya Puchka <ilyapuchka@gmail.com>
---
 Sources/Lexer.swift                | 16 ++++++++--------
 Tests/StencilTests/LexerSpec.swift | 10 ++++++++++
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/Sources/Lexer.swift b/Sources/Lexer.swift
index 47465f5..f6fe0a2 100644
--- a/Sources/Lexer.swift
+++ b/Sources/Lexer.swift
@@ -114,7 +114,7 @@ struct Lexer {
 class Scanner {
   let originalContent: String
   var content: String
-  var range: Range<String.Index>
+  var range: Range<String.UnicodeScalarView.Index>
 
   /// The start delimiter for a token.
   private static let tokenStartDelimiter: Unicode.Scalar = "{"
@@ -124,7 +124,7 @@ class Scanner {
   init(_ content: String) {
     self.originalContent = content
     self.content = content
-    range = content.startIndex..<content.startIndex
+    range = content.unicodeScalars.startIndex..<content.unicodeScalars.startIndex
   }
 
   var isEmpty: Bool {
@@ -147,9 +147,9 @@ class Scanner {
 
     for (index, char) in content.unicodeScalars.enumerated() {
       if foundChar && char == Scanner.tokenEndDelimiter {
-        let result = String(content.prefix(index + 1))
-        content = String(content.dropFirst(index + 1))
-        range = range.upperBound..<originalContent.index(range.upperBound, offsetBy: index + 1)
+        let result = String(content.unicodeScalars.prefix(index + 1))
+        content = String(content.unicodeScalars.dropFirst(index + 1))
+        range = range.upperBound..<originalContent.unicodeScalars.index(range.upperBound, offsetBy: index + 1)
         return result
       } else {
         foundChar = (char == tokenChar)
@@ -181,9 +181,9 @@ class Scanner {
     range = range.upperBound..<range.upperBound
     for (index, char) in content.unicodeScalars.enumerated() {
       if foundBrace && tokenChars.contains(char) {
-        let result = String(content.prefix(index - 1))
-        content = String(content.dropFirst(index - 1))
-        range = range.upperBound..<originalContent.index(range.upperBound, offsetBy: index - 1)
+        let result = String(content.unicodeScalars.prefix(index - 1))
+        content = String(content.unicodeScalars.dropFirst(index - 1))
+        range = range.upperBound..<originalContent.unicodeScalars.index(range.upperBound, offsetBy: index - 1)
         return (char, result)
       } else {
         foundBrace = (char == Scanner.tokenStartDelimiter)
diff --git a/Tests/StencilTests/LexerSpec.swift b/Tests/StencilTests/LexerSpec.swift
index adf6a92..60b8cf7 100644
--- a/Tests/StencilTests/LexerSpec.swift
+++ b/Tests/StencilTests/LexerSpec.swift
@@ -126,6 +126,16 @@ final class LexerTests: XCTestCase {
     }
   }
 
+  func testCombiningDiaeresis() throws {
+    // the symbol "ü" in the `templateString` is unusually encoded as 0x75 0xCC 0x88 (LATIN SMALL LETTER U + COMBINING DIAERESIS) instead of 0xC3 0xBC (LATIN SMALL LETTER U WITH DIAERESIS)
+    let templateString = "ü\n{% if test %}ü{% endif %}\n{% if ü %}ü{% endif %}\n"
+    let lexer = Lexer(templateString: templateString)
+    let tokens = lexer.tokenize()
+
+    try expect(tokens.count) == 9
+    assert(tokens[1].contents == "if test")
+  }
+
   private func makeSourceMap(_ token: String, for lexer: Lexer, options: String.CompareOptions = []) -> SourceMap {
     guard let range = lexer.templateString.range(of: token, options: options) else { fatalError("Token not found") }
     return SourceMap(location: lexer.rangeLocation(range))