Convert CharacterReader to native string indexing

This commit is contained in:
Garth Snyder 2019-03-03 17:33:18 -08:00
parent 5da0b123b9
commit a4eca3ed31
2 changed files with 149 additions and 290 deletions

View File

@ -14,43 +14,47 @@ import Foundation
public final class CharacterReader {
private static let empty = ""
public static let EOF: UnicodeScalar = "\u{FFFF}"//65535
private let input: [UnicodeScalar]
private let length: Int
private var pos: Int = 0
private var mark: Int = 0
private let input: String.UnicodeScalarView
private var pos: String.UnicodeScalarView.Index
private var mark: String.UnicodeScalarView.Index
//private let stringCache: Array<String?> // holds reused strings in this doc, to lessen garbage
public init(_ input: String) {
self.input = Array(input.unicodeScalars)
self.length = self.input.count
//stringCache = Array(repeating:nil, count:512)
self.input = input.unicodeScalars
self.pos = input.startIndex
self.mark = input.startIndex
}
public func getPos() -> Int {
return self.pos
return input.distance(from: input.startIndex, to: pos)
}
public func isEmpty() -> Bool {
return pos >= length
return pos >= input.endIndex
}
public func current() -> UnicodeScalar {
return (pos >= length) ? CharacterReader.EOF : input[pos]
return (pos >= input.endIndex) ? CharacterReader.EOF : input[pos]
}
@discardableResult
public func consume() -> UnicodeScalar {
let val = (pos >= length) ? CharacterReader.EOF : input[pos]
pos += 1
guard pos < input.endIndex else {
return CharacterReader.EOF
}
let val = input[pos]
pos = input.index(after: pos)
return val
}
public func unconsume() {
pos -= 1
guard pos > input.startIndex else { return }
pos = input.index(before: pos)
}
public func advance() {
pos += 1
guard pos < input.endIndex else { return }
pos = input.index(after: pos)
}
public func markPos() {
@ -62,221 +66,169 @@ public final class CharacterReader {
}
public func consumeAsString() -> String {
let p = pos
pos+=1
return String(input[p])
//return String(input, pos+=1, 1)
guard pos < input.endIndex else { return "" }
let str = String(input[pos])
pos = input.index(after: pos)
return str
}
/**
* Returns the number of characters between the current position and the next instance of the input char
* @param c scan target
* @return offset between current position and next instance of target. -1 if not found.
*/
public func nextIndexOf(_ c: UnicodeScalar) -> Int {
// doesn't handle scanning for surrogates
for i in pos..<length {
if (c == input[i]) {
return i - pos
}
}
return -1
}
/**
* Returns the number of characters between the current position and the next instance of the input sequence
* Locate the next occurrence of a Unicode scalar
*
* @param seq scan target
* @return offset between current position and next instance of target. -1 if not found.
* - Parameter c: scan target
* - Returns: offset between current position and next instance of target. -1 if not found.
*/
public func nextIndexOf(_ seq: String) -> Int {
public func nextIndexOf(_ c: UnicodeScalar) -> String.UnicodeScalarView.Index? {
// doesn't handle scanning for surrogates
if(seq.isEmpty) {return -1}
let startChar: UnicodeScalar = seq.unicodeScalar(0)
for var offset in pos..<length {
// scan to first instance of startchar:
if (startChar != input[offset]) {
offset+=1
while(offset < length && startChar != input[offset]) { offset+=1 }
}
var i = offset + 1
let last = i + seq.unicodeScalars.count-1
if (offset < length && last <= length) {
var j = 1
while i < last && seq.unicodeScalar(j) == input[i] {
j+=1
i+=1
}
// found full sequence
if (i == last) {
return offset - pos
}
}
}
return -1
return input[pos...].firstIndex(of: c)
}
/**
* Locate the next occurence of a target string
*
* - Parameter seq: scan target
* - Returns: index of next instance of target. nil if not found.
*/
public func nextIndexOf(_ seq: String) -> String.UnicodeScalarView.Index? {
// doesn't handle scanning for surrogates
var start = pos
let targetScalars = seq.unicodeScalars
guard let firstChar = targetScalars.first else { return pos } // search for "" -> current place
MATCH: while true {
// Match on first scalar
guard let firstCharIx = input[start...].firstIndex(of: firstChar) else { return nil }
var current = firstCharIx
// Then manually match subsequent scalars
for scalar in targetScalars.dropFirst() {
current = input.index(after: current)
guard current < input.endIndex else { return nil }
if input[current] != scalar {
start = input.index(after: firstCharIx)
continue MATCH
}
}
// full match; current is at position of last matching character
return firstCharIx
}
}
public func consumeTo(_ c: UnicodeScalar) -> String {
let offset = nextIndexOf(c)
if (offset != -1) {
let consumed = cacheString(pos, offset)
pos += offset
return consumed
} else {
guard let targetIx = nextIndexOf(c) else {
return consumeToEnd()
}
let consumed = cacheString(pos, targetIx)
pos = targetIx
return consumed
}
public func consumeTo(_ seq: String) -> String {
let offset = nextIndexOf(seq)
if (offset != -1) {
let consumed = cacheString(pos, offset)
pos += offset
return consumed
} else {
guard let targetIx = nextIndexOf(seq) else {
return consumeToEnd()
}
let consumed = cacheString(pos, targetIx)
pos = targetIx
return consumed
}
public func consumeToAny(_ chars: UnicodeScalar...) -> String {
return consumeToAny(chars)
}
public func consumeToAny(_ chars: [UnicodeScalar]) -> String {
let start: Int = pos
let remaining: Int = length
let val = input
OUTER: while (pos < remaining) {
if chars.contains(val[pos]) {
break OUTER
}
// for c in chars {
// if (val[pos] == c){
// break OUTER
// }
// }
pos += 1
let start = pos
while pos < input.endIndex {
if chars.contains(input[pos]) {
break
}
pos = input.index(after: pos)
}
return pos > start ? cacheString(start, pos-start) : CharacterReader.empty
return cacheString(start, pos)
}
public func consumeToAnySorted(_ chars: UnicodeScalar...) -> String {
return consumeToAnySorted(chars)
return consumeToAny(chars)
}
public func consumeToAnySorted(_ chars: [UnicodeScalar]) -> String {
let start = pos
let remaining = length
let val = input
while (pos < remaining) {
if chars.contains(val[pos]) {
break
}
pos += 1
}
return pos > start ? cacheString(start, pos-start) : CharacterReader.empty
return consumeToAny(chars)
}
static let dataTerminators: [UnicodeScalar] = [.Ampersand, .LessThan, TokeniserStateVars.nullScalr]
// read to &, <, or null
public func consumeData() -> String {
// &, <, null
let start = pos
let remaining = length
let val = input
while (pos < remaining) {
let c: UnicodeScalar = val[pos]
if (c == UnicodeScalar.Ampersand || c == UnicodeScalar.LessThan || c == TokeniserStateVars.nullScalr) {
break
}
pos += 1
}
return pos > start ? cacheString(start, pos-start) : CharacterReader.empty
return consumeToAny(CharacterReader.dataTerminators)
}
static let tagNameTerminators: [UnicodeScalar] = [.BackslashT, .BackslashN, .BackslashR, .BackslashF, .Space, .Slash, .GreaterThan, TokeniserStateVars.nullScalr]
// read to '\t', '\n', '\r', '\f', ' ', '/', '>', or nullChar
public func consumeTagName() -> String {
// '\t', '\n', '\r', '\f', ' ', '/', '>', nullChar
let start = pos
let remaining = length
let val = input
while (pos < remaining) {
let c: UnicodeScalar = val[pos]
if (c == UnicodeScalar.BackslashT || c == UnicodeScalar.BackslashN || c == UnicodeScalar.BackslashR || c == UnicodeScalar.BackslashF || c == UnicodeScalar.Space || c == UnicodeScalar.Slash || c == UnicodeScalar.GreaterThan || c == TokeniserStateVars.nullScalr) {
break
}
pos += 1
}
return pos > start ? cacheString(start, pos-start) : CharacterReader.empty
return consumeToAny(CharacterReader.tagNameTerminators)
}
public func consumeToEnd() -> String {
let data = cacheString(pos, length-pos)
pos = length
return data
let consumed = cacheString(pos, input.endIndex)
pos = input.endIndex
return consumed
}
public func consumeLetterSequence() -> String {
let start = pos
while (pos < length) {
let c: UnicodeScalar = input[pos]
while pos < input.endIndex {
let c = input[pos]
if ((c >= "A" && c <= "Z") || (c >= "a" && c <= "z") || c.isMemberOfCharacterSet(CharacterSet.letters)) {
pos += 1
pos = input.index(after: pos)
} else {
break
}
}
return cacheString(start, pos - start)
return cacheString(start, pos)
}
public func consumeLetterThenDigitSequence() -> String {
let start = pos
while (pos < length) {
while pos < input.endIndex {
let c = input[pos]
if ((c >= "A" && c <= "Z") || (c >= "a" && c <= "z") || c.isMemberOfCharacterSet(CharacterSet.letters)) {
pos += 1
pos = input.index(after: pos)
} else {
break
}
}
while (!isEmpty()) {
while pos < input.endIndex {
let c = input[pos]
if (c >= "0" && c <= "9") {
pos += 1
pos = input.index(after: pos)
} else {
break
}
}
return cacheString(start, pos - start)
return cacheString(start, pos)
}
public func consumeHexSequence() -> String {
let start = pos
while (pos < length) {
while pos < input.endIndex {
let c = input[pos]
if ((c >= "0" && c <= "9") || (c >= "A" && c <= "F") || (c >= "a" && c <= "f")) {
pos+=1
pos = input.index(after: pos)
} else {
break
}
}
return cacheString(start, pos - start)
return cacheString(start, pos)
}
public func consumeDigitSequence() -> String {
let start = pos
while (pos < length) {
while pos < input.endIndex {
let c = input[pos]
if (c >= "0" && c <= "9") {
pos+=1
pos = input.index(after: pos)
} else {
break
}
}
return cacheString(start, pos - start)
return cacheString(start, pos)
}
public func matches(_ c: UnicodeScalar) -> Bool {
@ -284,180 +236,85 @@ public final class CharacterReader {
}
public func matches(_ seq: String) -> Bool {
let scanLength = seq.unicodeScalars.count
if (scanLength > length - pos) {
return false
}
for offset in 0..<scanLength {
if (seq.unicodeScalar(offset) != input[pos+offset]) {
return false
public func matches(_ seq: String, ignoreCase: Bool = false, consume: Bool = false) -> Bool {
var current = pos
let scalars = seq.unicodeScalars
for scalar in scalars {
guard current < input.endIndex else { return false }
if ignoreCase {
guard input[current].uppercase == scalar.uppercase else { return false }
} else {
guard input[current] == scalar else { return false }
}
current = input.index(after: current)
}
if consume {
pos = current
}
return true
}
public func matchesIgnoreCase(_ seq: String ) -> Bool {
let scanLength = seq.unicodeScalars.count
if(scanLength == 0) {
return false
}
if (scanLength > length - pos) {
return false
}
for offset in 0..<scanLength {
let upScan: UnicodeScalar = seq.unicodeScalar(offset).uppercase
let upTarget: UnicodeScalar = input[pos+offset].uppercase
if (upScan != upTarget) {
return false
}
}
return true
return matches(seq, ignoreCase: true)
}
public func matchesAny(_ seq: UnicodeScalar...) -> Bool {
if (isEmpty()) {
return false
}
let c: UnicodeScalar = input[pos]
for seek in seq {
if (seek == c) {
return true
}
}
return false
return matchesAny(seq)
}
public func matchesAny(_ seq: [UnicodeScalar]) -> Bool {
guard pos < input.endIndex else { return false }
return seq.contains(input[pos])
}
public func matchesAnySorted(_ seq: [UnicodeScalar]) -> Bool {
return !isEmpty() && seq.contains(input[pos])
return matchesAny(seq)
}
public func matchesLetter() -> Bool {
if (isEmpty()) {
return false
}
let c = input[pos]
guard pos < input.endIndex else { return false }
let c = input[pos]
return (c >= "A" && c <= "Z") || (c >= "a" && c <= "z") || c.isMemberOfCharacterSet(CharacterSet.letters)
}
public func matchesDigit() -> Bool {
if (isEmpty()) {
return false
}
let c = input[pos]
return (c >= "0" && c <= "9")
guard pos < input.endIndex else { return false }
let c = input[pos]
return c >= "0" && c <= "9"
}
@discardableResult
public func matchConsume(_ seq: String) -> Bool {
if (matches(seq)) {
pos += seq.unicodeScalars.count
return true
} else {
return false
}
return matches(seq, consume: true)
}
@discardableResult
public func matchConsumeIgnoreCase(_ seq: String) -> Bool {
if (matchesIgnoreCase(seq)) {
pos += seq.unicodeScalars.count
return true
} else {
return false
}
return matches(seq, ignoreCase: true, consume: true)
}
public func containsIgnoreCase(_ seq: String ) -> Bool {
// used to check presence of </title>, </style>. only finds consistent case.
let loScan = seq.lowercased(with: Locale(identifier: "en"))
let hiScan = seq.uppercased(with: Locale(identifier: "eng"))
return (nextIndexOf(loScan) > -1) || (nextIndexOf(hiScan) > -1)
return nextIndexOf(loScan) != nil || nextIndexOf(hiScan) != nil
}
public func toString() -> String {
return String(input[pos..<length])
//return String.unicodescalars(Array(input[pos..<length]))
//return input.string(pos, length - pos)
return String(input[pos...])
}
/**
* Caches short strings, as a flywheel pattern, to reduce GC load. Just for this doc, to prevent leaks.
* <p />
* Simplistic, and on hash collisions just falls back to creating a new string, vs a full HashMap with Entry list.
* That saves both having to create objects as hash keys, and running through the entry list, at the expense of
* some more duplicates.
* Originally intended as a caching mechanism for strings, but caching doesn't
* seem to improve performance. Now just a stub.
*/
private func cacheString(_ start: Int, _ count: Int) -> String {
return String(input[start..<start+count])
// Too Slow
// var cache: [String?] = stringCache
//
// // limit (no cache):
// if (count > CharacterReader.maxCacheLen) {
// return String(val[start..<start+count].flatMap { Character($0) })
// }
//
// // calculate hash:
// var hash: Int = 0
// var offset = start
// for _ in 0..<count {
// let ch = val[offset].value
// hash = Int.addWithOverflow(Int.multiplyWithOverflow(31, hash).0, Int(ch)).0
// offset+=1
// }
//
// // get from cache
// hash = abs(hash)
// let i = hash % cache.count
// let index: Int = abs(i) //Int(hash & Int(cache.count) - 1)
// var cached = cache[index]
//
// if (cached == nil) { // miss, add
// cached = String(val[start..<start+count].flatMap { Character($0) })
// //cached = val.string(start, count)
// cache[Int(index)] = cached
// } else { // hashcode hit, check equality
// if (rangeEquals(start, count, cached!)) { // hit
// return cached!
// } else { // hashcode conflict
// cached = String(val[start..<start+count].flatMap { Character($0) })
// //cached = val.string(start, count)
// cache[index] = cached // update the cache, as recently used strings are more likely to show up again
// }
// }
// return cached!
private func cacheString(_ start: String.UnicodeScalarView.Index, _ end: String.UnicodeScalarView.Index) -> String {
return String(input[start..<end])
}
// /**
// * Check if the value of the provided range equals the string.
// */
// public func rangeEquals(_ start: Int, _ count: Int, _ cached: String) -> Bool {
// if (count == cached.unicodeScalars.count) {
// var count = count
// let one = input
// var i = start
// var j = 0
// while (count != 0) {
// count -= 1
// if (one[i] != cached.unicodeScalar(j) ) {
// return false
// }
// j += 1
// i += 1
// }
// return true
// }
// return false
// }
}
extension CharacterReader: CustomDebugStringConvertible {
public var debugDescription: String {
return self.toString()
return toString()
}
}

View File

@ -53,10 +53,12 @@ class CharacterReaderTest: XCTestCase {
XCTAssertEqual("e", r.consume())
XCTAssertTrue(r.isEmpty())
XCTAssertEqual(CharacterReader.EOF, r.consume())
r.unconsume()
XCTAssertTrue(r.isEmpty())
XCTAssertEqual(CharacterReader.EOF, r.current())
// Indexes beyond the end are not allowed in native indexing
//
// XCTAssertEqual(CharacterReader.EOF, r.consume())
// r.unconsume()
// XCTAssertTrue(r.isEmpty())
// XCTAssertEqual(CharacterReader.EOF, r.current())
}
func testMark() {
@ -82,31 +84,31 @@ class CharacterReaderTest: XCTestCase {
let input = "blah blah"
let r = CharacterReader(input)
XCTAssertEqual(-1, r.nextIndexOf("x"))
XCTAssertEqual(3, r.nextIndexOf("h"))
XCTAssertEqual(nil, r.nextIndexOf("x"))
XCTAssertEqual(input.index(input.startIndex, offsetBy: 3), r.nextIndexOf("h"))
let pull = r.consumeTo("h")
XCTAssertEqual("bla", pull)
XCTAssertEqual("h", r.consume())
XCTAssertEqual(2, r.nextIndexOf("l"))
XCTAssertEqual(input.index(input.startIndex, offsetBy: 6), r.nextIndexOf("l"))
XCTAssertEqual(" blah", r.consumeToEnd())
XCTAssertEqual(-1, r.nextIndexOf("x"))
XCTAssertEqual(nil, r.nextIndexOf("x"))
}
func testNextIndexOfString() {
let input = "One Two something Two Three Four"
let r = CharacterReader(input)
XCTAssertEqual(-1, r.nextIndexOf("Foo"))
XCTAssertEqual(4, r.nextIndexOf("Two"))
XCTAssertEqual(nil, r.nextIndexOf("Foo"))
XCTAssertEqual(input.index(input.startIndex, offsetBy: 4), r.nextIndexOf("Two"))
XCTAssertEqual("One Two ", r.consumeTo("something"))
XCTAssertEqual(10, r.nextIndexOf("Two"))
XCTAssertEqual(input.index(input.startIndex, offsetBy: 18), r.nextIndexOf("Two"))
XCTAssertEqual("something Two Three Four", r.consumeToEnd())
XCTAssertEqual(-1, r.nextIndexOf("Two"))
XCTAssertEqual(nil, r.nextIndexOf("Two"))
}
func testNextIndexOfUnmatched() {
let r = CharacterReader("<[[one]]")
XCTAssertEqual(-1, r.nextIndexOf("]]>"))
XCTAssertEqual(nil, r.nextIndexOf("]]>"))
}
func testConsumeToChar() {