swiftstencil/swiftpm-stencil
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Fix for incorrect tokenization due to index difference of Unicode character/scalar (#286)

Browse Source 
* Fix: `Scanner` now uses indices of the respective UnicodeScalarView

* Fix: `Scanner` now uses indices of the respective UnicodeScalarView

* Extended test for Unicode `Combining Diaeresis`

* Fixed test for combining diaeresis

* Inlined template for testing Unicode combining diaeresis

Co-authored-by: Ilya Puchka <ilyapuchka@gmail.com>
This commit is contained in:
Andreas Ley
2020-01-12 16:21:02 +01:00
committed by Ilya Puchka
parent 5b2d5dc5e0
commit a1718ae350
2 changed files with 18 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Tests/StencilTests/LexerSpec.swift
View File

@@ -126,6 +126,16 @@ final class LexerTests: XCTestCase {
}
}
func testCombiningDiaeresis() throws {
// the symbol "ü" in the `templateString` is unusually encoded as 0x75 0xCC 0x88 (LATIN SMALL LETTER U + COMBINING DIAERESIS) instead of 0xC3 0xBC (LATIN SMALL LETTER U WITH DIAERESIS)
let templateString = "ü\n{% if test %}ü{% endif %}\n{% if ü %}ü{% endif %}\n"
let lexer = Lexer(templateString: templateString)
let tokens = lexer.tokenize()
try expect(tokens.count) == 9
assert(tokens[1].contents == "if test")
}
private func makeSourceMap(_ token: String, for lexer: Lexer, options: String.CompareOptions = []) -> SourceMap {
guard let range = lexer.templateString.range(of: token, options: options) else { fatalError("Token not found") }
return SourceMap(location: lexer.rangeLocation(range))