dnrops
/
SwiftyMarkdown
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Completely overhauls tokenising engine

This commit is contained in:
Simon Fairbairn 2020-02-03 14:18:15 +13:00
parent 711850056d
commit 71eb29ada0
10 changed files with 927 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

285
Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Contents.swift Normal file
View File

@ -0,0 +1,285 @@
//: [Previous](@previous)
import Foundation
extension String {
func repeating( _ max : Int ) -> String {
var output = self
for _ in 1..<max {
output += self
}
return output
}
}
enum TagState {
case none
case open
case intermediate
case closed
}
struct TagString {
var state : TagState = .none
var preOpenString = ""
var openTagString = ""
var intermediateString = ""
var intermediateTagString = ""
var metadataString = ""
var closedTagString = ""
var postClosedString = ""
let rule : Rule
init( with rule : Rule ) {
self.rule = rule
}
mutating func append( _ string : String? ) {
guard let existentString = string else {
return
}
switch self.state {
case .none:
self.preOpenString += existentString
case .open:
self.intermediateString += existentString
case .intermediate:
self.metadataString += existentString
case .closed:
self.postClosedString += existentString
}
}
mutating func append( contentsOf tokenGroup: [TokenGroup] ) {
print(tokenGroup)
for token in tokenGroup {
switch token.state {
case .none:
self.append(token.string)
case .open:
if self.state != .none {
self.preOpenString += token.string
} else {
self.openTagString += token.string
}
case .intermediate:
if self.state != .open {
self.intermediateString += token.string
} else {
self.intermediateTagString += token.string
}
case .closed:
if self.rule.intermediateTag != nil && self.state != .intermediate {
self.metadataString += token.string
} else {
self.closedTagString += token.string
}
}
self.state = token.state
}
}
mutating func tokens() -> [Token] {
print(self)
var tokens : [Token] = []
if !self.preOpenString.isEmpty {
tokens.append(Token(type: .string, inputString: self.preOpenString))
}
if !self.openTagString.isEmpty {
tokens.append(Token(type: .openTag, inputString: self.openTagString))
}
if !self.intermediateString.isEmpty {
var token = Token(type: .string, inputString: self.intermediateString)
token.metadataString = self.metadataString
tokens.append(token)
}
if !self.intermediateTagString.isEmpty {
tokens.append(Token(type: .intermediateTag, inputString: self.intermediateTagString))
}
if !self.metadataString.isEmpty {
tokens.append(Token(type: .metadata, inputString: self.metadataString))
}
if !self.closedTagString.isEmpty {
tokens.append(Token(type: .closeTag, inputString: self.closedTagString))
}
self.preOpenString = ""
self.openTagString = ""
self.intermediateString = ""
self.intermediateTagString = ""
self.metadataString = ""
self.closedTagString = ""
self.postClosedString = ""
self.state = .none
return tokens
}
}
struct TokenGroup {
enum TokenGroupType {
case string
case tag
case escape
}
let string : String
let isEscaped : Bool
let type : TokenGroupType
var state : TagState = .none
}
func getTokenGroups( for string : inout String, with rule : Rule, shouldEmpty : Bool = false ) -> [TokenGroup] {
if string.isEmpty {
return []
}
let maxCount = rule.openTag.count * rule.maxTags
var groups : [TokenGroup] = []
let maxTag = rule.openTag.repeating(rule.maxTags)
if maxTag.contains(string) {
if string.count == maxCount || shouldEmpty {
var token = TokenGroup(string: string, isEscaped: false, type: .tag)
token.state = .open
groups.append(token)
string.removeAll()
}
} else if string == rule.intermediateTag {
var token = TokenGroup(string: string, isEscaped: false, type: .tag)
token.state = .intermediate
groups.append(token)
string.removeAll()
} else if string == rule.closingTag {
var token = TokenGroup(string: string, isEscaped: false, type: .tag)
token.state = .closed
groups.append(token)
string.removeAll()
}
if shouldEmpty && !string.isEmpty {
let token = TokenGroup(string: string, isEscaped: false, type: .tag)
groups.append(token)
string.removeAll()
}
return groups
}
func scan( _ string : String, with rule : Rule) -> [Token] {
let scanner = Scanner(string: string)
scanner.charactersToBeSkipped = nil
var tokens : [Token] = []
var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
if let existentEscape = rule.escapeCharacter {
set.insert(charactersIn: String(existentEscape))
}
var openTag = rule.openTag.repeating(rule.maxTags)
var tagString = TagString(with: rule)
var openTagFound : TagState = .none
var regularCharacters = ""
var tagGroupCount = 0
while !scanner.isAtEnd {
tagGroupCount += 1
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
if let start = scanner.scanUpToCharacters(from: set) {
tagString.append(start)
}
} else {
var string : NSString?
scanner.scanUpToCharacters(from: set, into: &string)
if let existentString = string as String? {
tagString.append(existentString)
}
}
// The end of the string
let maybeFoundChars = scanner.scanCharacters(from: set )
guard let foundTag = maybeFoundChars else {
continue
}
if foundTag == rule.openTag && foundTag.count < rule.minTags {
tagString.append(foundTag)
continue
}
//:--
print(foundTag)
var tokenGroups : [TokenGroup] = []
var escapeCharacter : Character? = nil
var cumulatedString = ""
for char in foundTag {
if let existentEscapeCharacter = escapeCharacter {
// If any of the tags feature the current character
let escape = String(existentEscapeCharacter)
let nextTagCharacter = String(char)
if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
escapeCharacter = nil
} else if nextTagCharacter == escape {
// Doesn't apply to this rule
tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
}
continue
}
if let existentEscape = rule.escapeCharacter {
if char == existentEscape {
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
escapeCharacter = char
continue
}
}
cumulatedString.append(char)
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule))
}
if let remainingEscape = escapeCharacter {
tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
}
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
tagString.append(contentsOf: tokenGroups)
if tagString.state == .closed {
tokens.append(contentsOf: tagString.tokens())
}
}
tokens.append(contentsOf: tagString.tokens())
return tokens
}
//: [Next](@next)
var string = "[]([[\\[Some Link]\\]](\\(\\(\\url) [Regular link](url)"
//string = "Text before [Regular link](url) Text after"
var output = "[]([[Some Link]] Regular link"
var tokens = scan(string, with: LinkRule())
print( tokens.filter( { $0.type == .string }).map({ $0.outputString }).joined())
//print( tokens )
//string = "**\\*\\Bold\\*\\***"
//output = "*\\Bold**"
//tokens = scan(string, with: AsteriskRule())
//print( tokens )

32
Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Sources/Tokens.swift Normal file
View File

@ -0,0 +1,32 @@
import Foundation
public protocol Rule {
var escapeCharacter : Character? { get }
var openTag : String { get }
var intermediateTag : String? { get }
var closingTag : String? { get }
var maxTags : Int { get }
var minTags : Int { get }
}
public struct LinkRule : Rule {
public let escapeCharacter : Character? = "\\"
public let openTag : String = "["
public let intermediateTag : String? = "]("
public let closingTag : String? = ")"
public let maxTags : Int = 1
public let minTags : Int = 1
public init() { }
}
public struct AsteriskRule : Rule {
public let escapeCharacter : Character? = "\\"
public let openTag : String = "*"
public let intermediateTag : String? = nil
public let closingTag : String? = nil
public let maxTags : Int = 3
public let minTags : Int = 1
public init() { }
}

1
Playground/SwiftyMarkdown.playground/Sources/Swifty Tokeniser.swift
View File

@ -74,6 +74,7 @@ public struct Token {
public let inputString : String
public var metadataString : String? = nil
public var characterStyles : [CharacterStyling] = []
public var group : Int = 0
public var count : Int = 0
public var shouldSkip : Bool = false
public var outputString : String {

2
Playground/SwiftyMarkdown.playground/contents.xcplayground
View File

@ -5,5 +5,7 @@
<page name='Line Processing'/>
<page name='Tokenising'/>
<page name='Attributed String'/>
<page name='SKLabelNode'/>
<page name='Groups'/>
</pages>
</playground>

65
Resources/test.md
View File

@ -62,3 +62,68 @@ Header 2
1. Including indented lists
- Up to three levels
1. Neat!
# SwiftyMarkdown 1.0
SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a Swift-style syntax. It uses dynamic type to set the font size correctly with whatever font you'd like to use.
## Fully Rebuilt For 2020!
SwiftyMarkdown now features a more robust and reliable rules-based line processing and tokenisation engine. It has added support for images stored in the bundle (`![Image](<Name In bundle>)`), codeblocks, blockquotes, and unordered lists!
Line-level attributes can now have a paragraph alignment applied to them (e.g. `h2.aligment = .center`), and links can be underlined by setting underlineLinks to `true`.
It also uses the system color `.label` as the default font color on iOS 13 and above for Dark Mode support out of the box.
## Installation
### CocoaPods:
`pod 'SwiftyMarkdown'`
### SPM:
In Xcode, `File -> Swift Packages -> Add Package Dependency` and add the GitHub URL.
*italics* or _italics_
**bold** or __bold__
~~Linethrough~~Strikethroughs.
`code`
# Header 1
or
Header 1
====
## Header 2
or
Header 2
---
### Header 3
#### Header 4
##### Header 5 #####
###### Header 6 ######
Indented code blocks (spaces or tabs)
[Links](http://voyagetravelapps.com/)
![Images](<Name of asset in bundle>)
> Blockquotes
- Bulleted
- Lists
- Including indented lists
- Up to three levels
- Neat!
1. Ordered
1. Lists
1. Including indented lists
- Up to three levels
1. Neat!

552
Sources/SwiftyMarkdown/SwiftyTokeniser.swift
View File

@ -28,9 +28,9 @@ public enum SpaceAllowed {
}
public enum Cancel {
case none
case allRemaining
case currentSet
case none
case allRemaining
case currentSet
}
public struct CharacterRule : CustomStringConvertible {
@ -44,6 +44,8 @@ public struct CharacterRule : CustomStringConvertible {
public var spacesAllowed : SpaceAllowed = .oneSide
public var cancels : Cancel = .none
public var tagVarieties : [Int : String]
public var description: String {
return "Character Rule with Open tag: \(self.openTag) and current styles : \(self.styles) "
}
@ -57,6 +59,11 @@ public struct CharacterRule : CustomStringConvertible {
self.minTags = minTags
self.maxTags = maxTags
self.cancels = cancels
self.tagVarieties = [:]
for i in minTags...maxTags {
self.tagVarieties[i] = openTag.repeating(i)
}
}
}
@ -77,6 +84,7 @@ public struct Token {
public let id = UUID().uuidString
public let type : TokenType
public let inputString : String
public fileprivate(set) var group : Int = 0
public fileprivate(set) var metadataString : String? = nil
public fileprivate(set) var characterStyles : [CharacterStyling] = []
public fileprivate(set) var count : Int = 0
@ -84,6 +92,8 @@ public struct Token {
public fileprivate(set) var tokenIndex : Int = -1
public fileprivate(set) var isProcessed : Bool = false
public fileprivate(set) var isMetadata : Bool = false
public var outputString : String {
get {
switch self.type {
@ -107,6 +117,9 @@ public struct Token {
self.type = type
self.inputString = inputString
self.characterStyles = characterStyles
if type == .repeatingTag {
self.count = inputString.count
}
}
func newToken( fromSubstring string: String, isReplacement : Bool) -> Token {
@ -119,15 +132,220 @@ public struct Token {
}
extension Sequence where Iterator.Element == Token {
var oslogDisplay: String {
var oslogDisplay: String {
return "[\"\(self.map( { ($0.outputString.isEmpty) ? "\($0.type): \($0.inputString)" : $0.outputString }).joined(separator: "\", \""))\"]"
}
}
}
enum TagState {
case none
case open
case intermediate
case closed
}
struct TagString {
var state : TagState = .none
var preOpenString = ""
var openTagString : [String] = []
var intermediateString = ""
var intermediateTagString = ""
var metadataString = ""
var closedTagString : [String] = []
var postClosedString = ""
let rule : CharacterRule
var tokenGroup = 0
init( with rule : CharacterRule ) {
self.rule = rule
}
mutating func append( _ string : String? ) {
guard let existentString = string else {
return
}
switch self.state {
case .none:
self.preOpenString += existentString
case .open:
self.intermediateString += existentString
case .intermediate:
self.metadataString += existentString
case .closed:
self.postClosedString += existentString
}
}
mutating func append( contentsOf tokenGroup: [TokenGroup] ) {
print(tokenGroup)
var availableCount = 0
for token in tokenGroup {
switch token.state {
case .none:
self.append(token.string)
if self.state == .closed {
self.state = .none
}
case .open:
switch self.state {
case .none:
self.openTagString.append(token.string)
self.state = .open
availableCount = self.rule.maxTags - 1
case .open:
if self.rule.closingTag == nil {
if availableCount > 0 {
self.openTagString.append(token.string)
availableCount -= 1
} else {
self.closedTagString.append(token.string)
self.state = .closed
}
} else if self.rule.maxTags == 1, self.openTagString.first == rule.openTag {
self.preOpenString = self.preOpenString + self.openTagString.joined() + self.intermediateString
self.intermediateString = ""
self.openTagString.append(token.string)
} else {
self.openTagString.append(token.string)
}
case .intermediate:
self.preOpenString += self.openTagString.joined() + token.string
case .closed:
self.openTagString.append(token.string)
}
case .intermediate:
switch self.state {
case .none:
self.preOpenString += token.string
case .open:
self.intermediateTagString += token.string
self.state = .intermediate
case .intermediate:
self.metadataString += token.string
case .closed:
self.postClosedString += token.string
}
case .closed:
switch self.state {
case .intermediate:
self.closedTagString.append(token.string)
self.state = .closed
case .closed:
self.postClosedString += token.string
case .open:
if self.rule.intermediateTag == nil {
self.closedTagString.append(token.string)
self.state = .closed
} else {
self.preOpenString += self.openTagString.joined()
self.preOpenString += self.intermediateString
self.preOpenString += token.string
self.intermediateString = ""
self.openTagString.removeAll()
}
case .none:
self.preOpenString += token.string
}
}
}
if !self.openTagString.isEmpty && self.rule.closingTag == nil && self.state != .closed {
self.state = .open
}
}
func configureToken(ofType type : TokenType = .string, with string : String ) -> Token {
var token = Token(type: type, inputString: string)
token.group = self.tokenGroup
return token
}
mutating func reset() {
self.preOpenString = ""
self.openTagString.removeAll()
self.intermediateString = ""
self.intermediateTagString = ""
self.metadataString = ""
self.closedTagString.removeAll()
self.postClosedString = ""
self.state = .none
}
mutating func tokens(beginningGroupNumberAt group : Int = 0) -> [Token] {
print(self)
self.tokenGroup = group
var tokens : [Token] = []
if self.intermediateString.isEmpty && self.intermediateTagString.isEmpty && self.metadataString.isEmpty {
tokens.append(self.configureToken(with: self.preOpenString + self.openTagString.joined() + self.closedTagString.joined() + self.postClosedString))
self.reset()
return tokens
}
if !self.preOpenString.isEmpty {
tokens.append(self.configureToken(with: self.preOpenString))
}
for tag in self.openTagString {
if self.rule.closingTag == nil {
tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
} else {
tokens.append(self.configureToken(ofType: .openTag, with: tag))
}
}
self.tokenGroup += 1
if !self.intermediateString.isEmpty {
var token = self.configureToken(with: self.intermediateString)
token.metadataString = self.metadataString
tokens.append(token)
}
if !self.intermediateTagString.isEmpty {
tokens.append(self.configureToken(ofType: .intermediateTag, with: self.intermediateTagString))
}
self.tokenGroup += 1
if !self.metadataString.isEmpty {
tokens.append(self.configureToken(with: self.metadataString))
}
for tag in self.closedTagString {
if self.rule.closingTag == nil {
tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
} else {
tokens.append(self.configureToken(ofType: .closeTag, with: tag))
}
}
if !self.postClosedString.isEmpty {
tokens.append(self.configureToken(with: self.postClosedString))
}
self.reset()
return tokens
}
}
struct TokenGroup {
enum TokenGroupType {
case string
case tag
case escape
}
let string : String
let isEscaped : Bool
let type : TokenGroupType
var state : TagState = .none
}
public class SwiftyTokeniser {
let rules : [CharacterRule]
var replacements : [String : [Token]] = [:]
var timer : TimeInterval = 0
var enableLog = (ProcessInfo.processInfo.environment["SwiftyTokeniserLogging"] != nil)
public init( with rules : [CharacterRule] ) {
@ -150,11 +368,12 @@ public class SwiftyTokeniser {
guard rules.count > 0 else {
return [Token(type: .string, inputString: inputString)]
}
var currentTokens : [Token] = []
var mutableRules = self.rules
self.timer = Date().timeIntervalSinceReferenceDate
// os_log("TIMER BEGIN: 0", log: .tokenising, type: .info)
while !mutableRules.isEmpty {
let nextRule = mutableRules.removeFirst()
@ -163,7 +382,7 @@ public class SwiftyTokeniser {
os_log("------------------------------", log: .tokenising, type: .info)
os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description)
}
if currentTokens.isEmpty {
// This means it's the first time through
currentTokens = self.applyStyles(to: self.scan(inputString, with: nextRule), usingRule: nextRule)
@ -224,7 +443,7 @@ public class SwiftyTokeniser {
// Each string could have additional tokens within it, so they have to be scanned as well with the current rule.
// The one string token might then be exploded into multiple more tokens
}
if enableLog {
os_log("=====RULE PROCESSING COMPLETE=====", log: .tokenising, type: .info)
os_log("==================================", log: .tokenising, type: .info)
@ -389,7 +608,7 @@ public class SwiftyTokeniser {
/// - incomingTokens: A group of tokens whose string tokens and replacement tokens should be combined and re-tokenised
/// - rule: The current rule being processed
func handleReplacementTokens( _ incomingTokens : [Token], with rule : CharacterRule) -> [Token] {
// Only combine string and replacements that are next to each other.
var newTokenSet : [Token] = []
var currentTokenSet : [Token] = []
@ -409,7 +628,7 @@ public class SwiftyTokeniser {
currentTokenSet.append(incomingTokens[i])
}
newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
return newTokenSet
}
@ -439,7 +658,7 @@ public class SwiftyTokeniser {
}
}
}
var metadataString : String = ""
for i in metadataIndex..<closeTokenIdx {
if tokens[i].type == .string {
@ -475,7 +694,7 @@ public class SwiftyTokeniser {
let theToken = tokens[index]
if enableLog {
os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
}
guard theToken.count > 0 else {
@ -487,7 +706,7 @@ public class SwiftyTokeniser {
let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
// Try to find exact match first
if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed }) {
if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) {
endIdx = nextTokenIdx
}
@ -534,9 +753,9 @@ public class SwiftyTokeniser {
let token = mutableTokens[idx]
switch token.type {
case .escape:
if enableLog {
os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
}
if enableLog {
os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
}
case .repeatingTag:
let theToken = mutableTokens[idx]
self.handleClosingTagFromRepeatingTag(withIndex: idx, in: &mutableTokens, following: rule)
@ -548,7 +767,7 @@ public class SwiftyTokeniser {
if enableLog {
os_log("Found open tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
}
guard rule.closingTag != nil else {
// If there's an intermediate tag, get the index of that
@ -568,8 +787,8 @@ public class SwiftyTokeniser {
case .closeTag:
let theToken = mutableTokens[idx]
if enableLog {
os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
if enableLog {
os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
}
case .string:
@ -594,13 +813,89 @@ public class SwiftyTokeniser {
return mutableTokens
}
enum TagState {
case open
case intermediate
case closed
func scanSpacing( _ scanner : Scanner, usingCharactersIn set : CharacterSet ) -> (preTag : String?, foundChars : String?, postTag : String?) {
let lastChar : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
lastChar = ( scanner.currentIndex > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
} else {
if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
lastChar = ( scanLocation > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanLocation)..<scanLocation]) : nil
} else {
lastChar = nil
}
}
let maybeFoundChars : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
maybeFoundChars = scanner.scanCharacters(from: set )
} else {
var string : NSString?
scanner.scanCharacters(from: set, into: &string)
maybeFoundChars = string as String?
}
let nextChar : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0,tvOS 13.0, *) {
nextChar = (scanner.currentIndex != scanner.string.endIndex) ? String(scanner.string[scanner.currentIndex]) : nil
} else {
if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
nextChar = (scanLocation != scanner.string.endIndex) ? String(scanner.string[scanLocation]) : nil
} else {
nextChar = nil
}
}
return (lastChar, maybeFoundChars, nextChar)
}
func getTokenGroups( for string : inout String, with rule : CharacterRule, shouldEmpty : Bool = false ) -> [TokenGroup] {
if string.isEmpty {
return []
}
var groups : [TokenGroup] = []
if string.contains(rule.openTag) {
if shouldEmpty || string == rule.tagVarieties[rule.maxTags]{
var token = TokenGroup(string: string, isEscaped: false, type: .tag)
token.state = .open
groups.append(token)
string.removeAll()
}
} else if let intermediateString = rule.intermediateTag, string.contains(intermediateString) {
if let range = string.range(of: intermediateString) {
let prior = string[string.startIndex..<range.lowerBound]
let tag = string[range]
let following = string[range.upperBound..<string.endIndex]
if !prior.isEmpty {
groups.append(TokenGroup(string: String(prior), isEscaped: false, type: .string))
}
var token = TokenGroup(string: String(tag), isEscaped: false, type: .tag)
token.state = .intermediate
groups.append(token)
if !following.isEmpty {
groups.append(TokenGroup(string: String(following), isEscaped: false, type: .string))
}
string.removeAll()
}
} else if let closingTag = rule.closingTag, closingTag.contains(string) {
var token = TokenGroup(string: string, isEscaped: false, type: .tag)
token.state = .closed
groups.append(token)
string.removeAll()
}
if shouldEmpty && !string.isEmpty {
let token = TokenGroup(string: string, isEscaped: false, type: .tag)
groups.append(token)
string.removeAll()
}
return groups
}
func scan( _ string : String, with rule : CharacterRule) -> [Token] {
os_log("TIMER CHECK: %f for rule with openTag %@", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg, rule.openTag)
let scanner = Scanner(string: string)
scanner.charactersToBeSkipped = nil
var tokens : [Token] = []
@ -610,192 +905,104 @@ public class SwiftyTokeniser {
}
var openTagFound : TagState = .open
var openingString = ""
var tagString = TagString(with: rule)
var tokenGroup = 0
while !scanner.isAtEnd {
tokenGroup += 1
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
if let start = scanner.scanUpToCharacters(from: set) {
openingString.append(start)
tagString.append(start)
}
} else {
var string : NSString?
scanner.scanUpToCharacters(from: set, into: &string)
if let existentString = string as String? {
openingString.append(existentString)
tagString.append(existentString)
}
// Fallback on earlier versions
}
let lastChar : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
lastChar = ( scanner.currentIndex > string.startIndex ) ? String(string[string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
} else {
if let scanLocation = string.index(string.startIndex, offsetBy: scanner.scanLocation, limitedBy: string.endIndex) {
lastChar = ( scanLocation > string.startIndex ) ? String(string[string.index(before: scanLocation)..<scanLocation]) : nil
} else {
lastChar = nil
}
}
let maybeFoundChars : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
maybeFoundChars = scanner.scanCharacters(from: set )
} else {
var string : NSString?
scanner.scanCharacters(from: set, into: &string)
maybeFoundChars = string as String?
}
// os_log("TIMER CHECK (pre-spacing): %f", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg)
let nextChar : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0,tvOS 13.0, *) {
nextChar = (scanner.currentIndex != string.endIndex) ? String(string[scanner.currentIndex]) : nil
} else {
if let scanLocation = string.index(string.startIndex, offsetBy: scanner.scanLocation, limitedBy: string.endIndex) {
nextChar = (scanLocation != string.endIndex) ? String(string[scanLocation]) : nil
} else {
nextChar = nil
}
}
// The end of the string
let spacing = self.scanSpacing(scanner, usingCharactersIn: set)
guard let foundChars = maybeFoundChars else {
tokens.append(Token(type: .string, inputString: "\(openingString)"))
openingString = ""
guard let foundTag = spacing.foundChars else {
continue
}
if foundChars == rule.openTag && foundChars.count < rule.minTags {
openingString.append(foundChars)
if foundTag == rule.openTag && foundTag.count < rule.minTags {
tagString.append(foundTag)
continue
}
if !validateSpacing(nextCharacter: nextChar, previousCharacter: lastChar, with: rule) {
if !validateSpacing(nextCharacter: spacing.postTag, previousCharacter: spacing.preTag, with: rule) {
let escapeString = String("\(rule.escapeCharacter ?? Character(""))")
var escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
var escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
if let hasIntermediateTag = rule.intermediateTag {
escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
}
if let existentClosingTag = rule.closingTag {
escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
}
openingString.append(escaped)
tagString.append(escaped)
continue
}
// Here's where we have to do the actual tag management.
var cumulativeString = ""
var openString = ""
var containedText = ""
var intermediateString = ""
var metadataText = ""
var closedString = ""
var maybeEscapeNext = false
// os_log("TIMER CHECK (pre grouping): %f", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg)
//:--
print(foundTag)
func addToken( for type : TokenType ) {
var inputString : String
switch type {
case .openTag:
inputString = openString
case .intermediateTag:
inputString = intermediateString
case .closeTag:
inputString = closedString
default:
inputString = ""
}
guard !inputString.isEmpty else {
return
}
guard inputString.count >= rule.minTags else {
return
}
if !openingString.isEmpty {
tokens.append(Token(type: .string, inputString: "\(openingString)"))
openingString = ""
}
let actualType : TokenType = ( rule.intermediateTag == nil && rule.closingTag == nil ) ? .repeatingTag : type
var token = Token(type: actualType, inputString: inputString)
if rule.closingTag == nil {
token.count = inputString.count
}
tokens.append(token)
switch type {
case .openTag:
openString = ""
case .intermediateTag:
intermediateString = ""
case .closeTag:
closedString = ""
default:
break
}
if !foundTag.contains(rule.openTag) && !foundTag.contains(rule.intermediateTag ?? "") && !foundTag.contains(rule.closingTag ?? "") {
tagString.append(foundTag)
continue
}
// Here I am going through and adding the characters in the found set to a cumulative string.
// If there is an escape character, then the loop stops and any open tags are tokenised.
for char in foundChars {
cumulativeString.append(char)
if maybeEscapeNext {
var tokenGroups : [TokenGroup] = []
var escapeCharacter : Character? = nil
var cumulatedString = ""
for char in foundTag {
if let existentEscapeCharacter = escapeCharacter {
var escaped = cumulativeString
if String(char) == rule.openTag || String(char) == rule.intermediateTag || String(char) == rule.closingTag {
escaped = String(cumulativeString.replacingOccurrences(of: String(rule.escapeCharacter ?? Character("")), with: ""))
// If any of the tags feature the current character
let escape = String(existentEscapeCharacter)
let nextTagCharacter = String(char)
if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
escapeCharacter = nil
} else if nextTagCharacter == escape {
// Doesn't apply to this rule
tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
}
openingString.append(escaped)
cumulativeString = ""
maybeEscapeNext = false
continue
}
if let existentEscape = rule.escapeCharacter {
if cumulativeString == String(existentEscape) {
maybeEscapeNext = true
addToken(for: .openTag)
addToken(for: .intermediateTag)
addToken(for: .closeTag)
if char == existentEscape {
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
escapeCharacter = char
continue
}
}
cumulatedString.append(char)
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule))
if cumulativeString == rule.openTag, openTagFound == .open {
openString.append(char)
cumulativeString = ""
openTagFound = ( rule.closingTag == nil ) ? .open : .closed
openTagFound = ( rule.intermediateTag == nil ) ? openTagFound : .intermediate
} else if cumulativeString == rule.intermediateTag, openTagFound == .intermediate {
intermediateString.append(cumulativeString)
cumulativeString = ""
openTagFound = ( rule.closingTag == nil ) ? .open : .closed
} else if cumulativeString == rule.closingTag, openTagFound == .closed {
closedString.append(char)
cumulativeString = ""
openTagFound = .open
}
}
if let remainingEscape = escapeCharacter {
tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
}
addToken(for: .openTag)
addToken(for: .intermediateTag)
addToken(for: .closeTag)
openingString.append( cumulativeString )
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
tagString.append(contentsOf: tokenGroups)
// If we're here, it means that an escape character was found but without a corresponding
// tag, which means it might belong to a different rule.
// It should be added to the next group of regular characters
if tagString.state == .none {
tokens.append(contentsOf: tagString.tokens(beginningGroupNumberAt : tokenGroup))
}
}
if !openingString.isEmpty {
tokens.append(Token(type: .string, inputString: "\(openingString)"))
}
tokens.append(contentsOf: tagString.tokens(beginningGroupNumberAt : tokenGroup))
// os_log("TIMER TOTAL: %f for rule with openTag %@", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg, rule.openTag)
return tokens
}
@ -823,7 +1030,7 @@ public class SwiftyTokeniser {
default:
return true
}
case .oneSide:
switch (previousCharacter, nextCharacter) {
case (nil, " " ), (" ", nil), (" ", " " ):
@ -838,3 +1045,14 @@ public class SwiftyTokeniser {
}
}
extension String {
func repeating( _ max : Int ) -> String {
var output = self
for _ in 1..<max {
output += self
}
return output
}
}

4
SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/AD1DF83E-20BC-4E7E-8C14-683818ED0A26.plist
View File

@ -11,9 +11,11 @@
<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
<dict>
<key>baselineAverage</key>
<real>0.0217</real>
<real>0.01</real>
<key>baselineIntegrationDisplayName</key>
<string>Local Baseline</string>
<key>maxPercentRelativeStandardDeviation</key>
<real>10</real>
</dict>
</dict>
<key>testThatStringsAreProcessedQuickly()</key>

82
Tests/SwiftyMarkdownTests/SwiftyMarkdownCharacterTests.swift
View File

@ -12,36 +12,50 @@ import XCTest
class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
func testIsolatedCase() {
let challenge = TokenTest(input: "An *\\*italic\\** [referenced link][link]", output: "An *italic* referenced link", tokens: [
challenge = TokenTest(input: "A string with a **bold** word", output: "A string with a bold word", tokens: [
Token(type: .string, inputString: "A string with a ", characterStyles: []),
Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
Token(type: .string, inputString: " word", characterStyles: [])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
return
challenge = TokenTest(input: "An *\\*italic\\** [referenced link][link]", output: "An *italic* referenced link", tokens: [
Token(type: .string, inputString: "An ", characterStyles: []),
Token(type: .string, inputString: "*italic*", characterStyles: [CharacterStyle.italic]),
Token(type: .string, inputString: " ", characterStyles: []),
Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])
])
let rules : [CharacterRule] = [
rules = [
CharacterRule(openTag: "*", escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [.bold], 3 : [.italic, .bold]], minTags: 1, maxTags: 3),
CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], minTags: 1, maxTags: 1),
CharacterRule(openTag: "[", intermediateTag: "][", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], minTags: 1, maxTags: 1)
]
let results = self.attempt(challenge, rules: rules)
results = self.attempt(challenge, rules: rules)
XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
if links.count == 1 {
XCTAssertEqual(links[0].metadataString, "https://www.neverendingvoyage.com/")
if results.links.count == 1 {
XCTAssertEqual(results.links[0].metadataString, "https://www.neverendingvoyage.com/")
} else {
XCTFail("Incorrect link count. Expecting 1, found \(links.count)")
XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
}
}
func testThatBoldTraitsAreRecognised() {
var challenge = TokenTest(input: "**A bold string**", output: "A bold string", tokens: [
challenge = TokenTest(input: "**A bold string**", output: "A bold string", tokens: [
Token(type: .string, inputString: "A bold string", characterStyles: [CharacterStyle.bold])
])
var results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
@ -67,6 +81,15 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
challenge = TokenTest(input: "\\\\*\\*A normal \\\\ string\\*\\*", output: "\\**A normal \\\\ string**", tokens: [
Token(type: .string, inputString: "\\**A normal \\\\ string**", characterStyles: [])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
challenge = TokenTest(input: "A string with double \\*\\*escaped\\*\\* asterisks", output: "A string with double **escaped** asterisks", tokens: [
Token(type: .string, inputString: "A string with double **escaped** asterisks", characterStyles: [])
])
@ -99,10 +122,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
}
func testThatCodeTraitsAreRecognised() {
var challenge = TokenTest(input: "`Code (**should** not process internal tags)`", output: "Code (**should** not process internal tags)", tokens: [
challenge = TokenTest(input: "`Code (**should** not process internal tags)`", output: "Code (**should** not process internal tags)", tokens: [
Token(type: .string, inputString: "Code (**should** not process internal tags) ", characterStyles: [CharacterStyle.code])
])
var results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
@ -158,14 +181,23 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
challenge = TokenTest(input: "Two backticks followed by a full stop ``.", output: "Two backticks followed by a full stop ``.", tokens: [
Token(type: .string, inputString: "Two backticks followed by a full stop ``.", characterStyles: [])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
}
func testThatItalicTraitsAreParsedCorrectly() {
var challenge = TokenTest(input: "*An italicised string*", output: "An italicised string", tokens : [
challenge = TokenTest(input: "*An italicised string*", output: "An italicised string", tokens : [
Token(type: .string, inputString: "An italicised string", characterStyles: [CharacterStyle.italic])
])
var results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
@ -236,11 +268,11 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
}
func testThatStrikethroughTraitsAreRecognised() {
var challenge = TokenTest(input: "~~An~~A crossed-out string", output: "AnA crossed-out string", tokens: [
challenge = TokenTest(input: "~~An~~A crossed-out string", output: "AnA crossed-out string", tokens: [
Token(type: .string, inputString: "An", characterStyles: [CharacterStyle.strikethrough]),
Token(type: .string, inputString: "A crossed-out string", characterStyles: [])
])
var results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
@ -269,7 +301,7 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
func testThatMixedTraitsAreRecognised() {
var challenge = TokenTest(input: "__A bold string__ with a **mix** **of** bold __styles__", output: "A bold string with a mix of bold styles", tokens : [
challenge = TokenTest(input: "__A bold string__ with a **mix** **of** bold __styles__", output: "A bold string with a mix of bold styles", tokens : [
Token(type: .string, inputString: "A bold string", characterStyles: [CharacterStyle.bold]),
Token(type: .string, inputString: "with a ", characterStyles: []),
Token(type: .string, inputString: "mix", characterStyles: [CharacterStyle.bold]),
@ -278,7 +310,7 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
Token(type: .string, inputString: " bold ", characterStyles: []),
Token(type: .string, inputString: "styles", characterStyles: [CharacterStyle.bold])
])
var results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
@ -306,10 +338,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
}
func testThatExtraCharactersAreHandles() {
var challenge = TokenTest(input: "***A bold italic string***", output: "A bold italic string", tokens: [
challenge = TokenTest(input: "***A bold italic string***", output: "A bold italic string", tokens: [
Token(type: .string, inputString: "A bold italic string", characterStyles: [CharacterStyle.bold, CharacterStyle.italic])
])
var results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
@ -380,12 +412,12 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
func offtestAdvancedEscaping() {
var challenge = TokenTest(input: "\\***A normal string*\\**", output: "**A normal string*", tokens: [
challenge = TokenTest(input: "\\***A normal string*\\**", output: "**A normal string*", tokens: [
Token(type: .string, inputString: "**", characterStyles: []),
Token(type: .string, inputString: "A normal string", characterStyles: [CharacterStyle.italic]),
Token(type: .string, inputString: "**", characterStyles: [])
])
var results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
@ -411,7 +443,6 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
let asteriskComma = "An asterisk followed by a full stop: *, *"
let backtickSpace = "A backtick followed by a space: `"
let backtickFullStop = "Two backticks followed by a full stop: ``."
let underscoreSpace = "An underscore followed by a space: _"
@ -431,9 +462,6 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
md = SwiftyMarkdown(string: asteriskFullStop)
XCTAssertEqual(md.attributedString().string, asteriskFullStop)
md = SwiftyMarkdown(string: backtickFullStop)
XCTAssertEqual(md.attributedString().string, backtickFullStop)
md = SwiftyMarkdown(string: underscoreFullStop)
XCTAssertEqual(md.attributedString().string, underscoreFullStop)
@ -458,10 +486,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
}
func testReportedCrashingStrings() {
let challenge = TokenTest(input: "[**\\!bang**](https://duckduckgo.com/bang)", output: "\\!bang", tokens: [
challenge = TokenTest(input: "[**\\!bang**](https://duckduckgo.com/bang)", output: "\\!bang", tokens: [
Token(type: .string, inputString: "\\!bang", characterStyles: [CharacterStyle.bold, CharacterStyle.link])
])
let results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)

94
Tests/SwiftyMarkdownTests/SwiftyMarkdownLinkTests.swift
View File

@ -13,10 +13,10 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
func testForLinks() {
var challenge = TokenTest(input: "[Link at start](http://voyagetravelapps.com/)", output: "Link at start", tokens: [
challenge = TokenTest(input: "[Link at start](http://voyagetravelapps.com/)", output: "Link at start", tokens: [
Token(type: .string, inputString: "Link at start", characterStyles: [CharacterStyle.link])
])
var results = self.attempt(challenge)
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
@ -82,25 +82,10 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
}
challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/)", tokens: [
Token(type: .string, inputString: "Link with missing square(http://voyagetravelapps.com/)", characterStyles: [])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/)", tokens: [
Token(type: .string, inputString: "A ", characterStyles: []),
Token(type: .string, inputString: "[Link(http://voyagetravelapps.com/)", characterStyles: [])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
}
func testMalformedLinks() {
challenge = TokenTest(input: "[Link with missing parenthesis](http://voyagetravelapps.com/", output: "[Link with missing parenthesis](http://voyagetravelapps.com/", tokens: [
Token(type: .string, inputString: "[Link with missing parenthesis](", characterStyles: []),
@ -123,19 +108,61 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
challenge = TokenTest(input: "[Link1](http://voyagetravelapps.com/) **bold** [Link2](http://voyagetravelapps.com/)", output: "Link1 bold Link2", tokens: [
Token(type: .string, inputString: "Link1", characterStyles: [CharacterStyle.link]),
Token(type: .string, inputString: " ", characterStyles: []),
Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
Token(type: .string, inputString: " ", characterStyles: []),
Token(type: .string, inputString: "Link2", characterStyles: [CharacterStyle.link])
challenge = TokenTest(input: "[A link](((url)", output: "A link", tokens: [
Token(type: .string, inputString: "A link", characterStyles: [CharacterStyle.link])
])
rules = [
CharacterRule(openTag: "![", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.image]]),
CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]]),
]
results = self.attempt(challenge, rules: rules)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
XCTAssertEqual(results.links.count, 1)
if results.links.count == 1 {
XCTAssertEqual(results.links[0].metadataString, "((url")
} else {
XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
}
challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/)", tokens: [
Token(type: .string, inputString: "Link with missing square(http://voyagetravelapps.com/)", characterStyles: [])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
challenge = TokenTest(input: "[Link with [second opening](http://voyagetravelapps.com/)", output: "[Link with second opening", tokens: [
Token(type: .string, inputString: "Link with ", characterStyles: []),
Token(type: .string, inputString: "second opening", characterStyles: [CharacterStyle.link])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
XCTAssertEqual(results.links.count, 1)
if results.links.count == 1 {
XCTAssertEqual(results.links[0].metadataString, "http://voyagetravelapps.com/")
} else {
XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
}
challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/)", tokens: [
Token(type: .string, inputString: "A [Link(http://voyagetravelapps.com/)", characterStyles: [])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
}
func testLinksWithOtherStyles() {
@ -173,6 +200,19 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
} else {
XCTFail("Incorrect link count. Expecting 1, found \(links.count)")
}
challenge = TokenTest(input: "[Link1](http://voyagetravelapps.com/) **bold** [Link2](http://voyagetravelapps.com/)", output: "Link1 bold Link2", tokens: [
Token(type: .string, inputString: "Link1", characterStyles: [CharacterStyle.link]),
Token(type: .string, inputString: " ", characterStyles: []),
Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
Token(type: .string, inputString: " ", characterStyles: []),
Token(type: .string, inputString: "Link2", characterStyles: [CharacterStyle.link])
])
results = self.attempt(challenge)
XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
}
func testForImages() {

52
Tests/SwiftyMarkdownTests/XCTest+SwiftyMarkdown.swift
View File

@ -9,11 +9,41 @@
import XCTest
@testable import SwiftyMarkdown
struct ChallengeReturn {
let tokens : [Token]
let stringTokens : [Token]
let links : [Token]
let attributedString : NSAttributedString
let foundStyles : [[CharacterStyle]]
let expectedStyles : [[CharacterStyle]]
}
class SwiftyMarkdownCharacterTests : XCTestCase {
let defaultRules = SwiftyMarkdown.characterRules
func testDummy() {
var challenge : TokenTest!
var results : ChallengeReturn!
var rules : [CharacterRule]? = nil
func attempt( _ challenge : TokenTest, rules : [CharacterRule]? = nil ) -> ChallengeReturn {
if let validRules = rules {
SwiftyMarkdown.characterRules = validRules
} else {
SwiftyMarkdown.characterRules = self.defaultRules
}
let md = SwiftyMarkdown(string: challenge.input)
let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
let tokens = tokeniser.process(challenge.input)
let stringTokens = tokens.filter({ $0.type == .string && !$0.isMetadata })
let existentTokenStyles = stringTokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
let expectedStyles = challenge.tokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
let linkTokens = tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
return ChallengeReturn(tokens: tokens, stringTokens: stringTokens, links : linkTokens, attributedString: md.attributedString(), foundStyles: existentTokenStyles, expectedStyles : expectedStyles)
}
}
@ -29,22 +59,4 @@ extension XCTestCase {
}
extension SwiftyMarkdownCharacterTests {
func attempt( _ challenge : TokenTest, rules : [CharacterRule]? = nil ) -> (tokens : [Token], stringTokens: [Token], attributedString : NSAttributedString, foundStyles : [[CharacterStyle]], expectedStyles : [[CharacterStyle]] ) {
if let validRules = rules {
SwiftyMarkdown.characterRules = validRules
} else {
SwiftyMarkdown.characterRules = self.defaultRules
}
let md = SwiftyMarkdown(string: challenge.input)
let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
let tokens = tokeniser.process(challenge.input)
let stringTokens = tokens.filter({ $0.type == .string && !$0.isMetadata })
let existentTokenStyles = stringTokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
let expectedStyles = challenge.tokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
return (tokens, stringTokens, md.attributedString(), existentTokenStyles, expectedStyles)
}
}