swiftstencil/swiftpm-stencil
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix for incorrect tokenization due to index difference of Unicode character/scalar (#286)

Browse Source 
* Fix: `Scanner` now uses indices of the respective UnicodeScalarView

* Fix: `Scanner` now uses indices of the respective UnicodeScalarView

* Extended test for Unicode `Combining Diaeresis`

* Fixed test for combining diaeresis

* Inlined template for testing Unicode combining diaeresis

Co-authored-by: Ilya Puchka <ilyapuchka@gmail.com>
This commit is contained in:
Andreas Ley
2020-01-12 16:21:02 +01:00
committed by Ilya Puchka
parent 5b2d5dc5e0
commit a1718ae350
2 changed files with 18 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Sources/Lexer.swift
View File

@@ -114,7 +114,7 @@ struct Lexer {
class Scanner {
let originalContent: String
var content: String
var range: Range<String.Index>
var range: Range<String.UnicodeScalarView.Index>
/// The start delimiter for a token.
private static let tokenStartDelimiter: Unicode.Scalar = "{"
@@ -124,7 +124,7 @@ class Scanner {
init(_ content: String) {
self.originalContent = content
self.content = content
range = content.startIndex..<content.startIndex
range = content.unicodeScalars.startIndex..<content.unicodeScalars.startIndex
}
var isEmpty: Bool {
@@ -147,9 +147,9 @@ class Scanner {
for (index, char) in content.unicodeScalars.enumerated() {
if foundChar && char == Scanner.tokenEndDelimiter {
let result = String(content.prefix(index + 1))
content = String(content.dropFirst(index + 1))
range = range.upperBound..<originalContent.index(range.upperBound, offsetBy: index + 1)
let result = String(content.unicodeScalars.prefix(index + 1))
content = String(content.unicodeScalars.dropFirst(index + 1))
range = range.upperBound..<originalContent.unicodeScalars.index(range.upperBound, offsetBy: index + 1)
return result
} else {
foundChar = (char == tokenChar)
@@ -181,9 +181,9 @@ class Scanner {
range = range.upperBound..<range.upperBound
for (index, char) in content.unicodeScalars.enumerated() {
if foundBrace && tokenChars.contains(char) {
let result = String(content.prefix(index - 1))
content = String(content.dropFirst(index - 1))
range = range.upperBound..<originalContent.index(range.upperBound, offsetBy: index - 1)
let result = String(content.unicodeScalars.prefix(index - 1))
content = String(content.unicodeScalars.dropFirst(index - 1))
range = range.upperBound..<originalContent.unicodeScalars.index(range.upperBound, offsetBy: index - 1)
return (char, result)
} else {
foundBrace = (char == Scanner.tokenStartDelimiter)