Completely overhauls tokenising engine

Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Contents.swift

@@ -0,0 +1,285 @@
+//: [Previous](@previous)
+
+import Foundation
+
+extension String {
+	func repeating( _ max : Int ) -> String {
+		var output = self
+		for _ in 1..<max {
+			output += self
+		}
+		return output
+	}
+}
+
+enum TagState {
+	case none
+	case open
+	case intermediate
+	case closed
+}
+
+struct TagString {
+	var state : TagState = .none
+	var preOpenString = ""
+	var openTagString = ""
+	var intermediateString = ""
+	var intermediateTagString = ""
+	var metadataString = ""
+	var closedTagString = ""
+	var postClosedString = ""
+
+	let rule : Rule
+	
+	init( with rule : Rule ) {
+		self.rule = rule
+	}
+	
+	mutating func append( _ string : String? ) {
+		guard let existentString = string else {
+			return
+		}
+		switch self.state {
+		case .none:
+			self.preOpenString += existentString
+		case .open:
+			self.intermediateString += existentString
+		case .intermediate:
+			self.metadataString += existentString
+		case .closed:
+			self.postClosedString += existentString
+		}
+	}
+	
+	mutating func append( contentsOf tokenGroup: [TokenGroup] ) {
+		print(tokenGroup)
+		for token in tokenGroup {
+			switch token.state {
+			case .none:
+				self.append(token.string)
+			case .open:
+				if self.state != .none {
+					self.preOpenString += token.string
+				} else {
+					self.openTagString += token.string
+				}
+			case .intermediate:
+				if self.state != .open {
+					self.intermediateString += token.string
+				} else {
+					self.intermediateTagString += token.string
+				}
+			case .closed:
+				if self.rule.intermediateTag != nil && self.state != .intermediate {
+					self.metadataString += token.string
+				} else {
+					self.closedTagString += token.string
+				}
+			}
+			self.state = token.state
+		}
+	}
+	
+	mutating func tokens() -> [Token] {
+		print(self)
+		var tokens : [Token] = []
+
+		if !self.preOpenString.isEmpty {
+			tokens.append(Token(type: .string, inputString: self.preOpenString))
+		}
+		if !self.openTagString.isEmpty {
+			tokens.append(Token(type: .openTag, inputString: self.openTagString))
+		}
+		if !self.intermediateString.isEmpty {
+			var token = Token(type: .string, inputString: self.intermediateString)
+			token.metadataString = self.metadataString
+			tokens.append(token)
+		}
+		if !self.intermediateTagString.isEmpty {
+			tokens.append(Token(type: .intermediateTag, inputString: self.intermediateTagString))
+		}
+		if !self.metadataString.isEmpty {
+			tokens.append(Token(type: .metadata, inputString: self.metadataString))
+		}
+		if !self.closedTagString.isEmpty {
+			tokens.append(Token(type: .closeTag, inputString: self.closedTagString))
+		}
+		
+		self.preOpenString = ""
+		self.openTagString = ""
+		self.intermediateString = ""
+		self.intermediateTagString = ""
+		self.metadataString = ""
+		self.closedTagString = ""
+		self.postClosedString = ""
+		
+		self.state = .none
+		
+		return tokens
+	}
+}
+
+struct TokenGroup {
+	enum TokenGroupType {
+		case string
+		case tag
+		case escape
+	}
+
+	let string : String
+	let isEscaped : Bool
+	let type : TokenGroupType
+	var state : TagState = .none
+}
+
+
+
+func getTokenGroups( for string : inout String, with rule : Rule, shouldEmpty : Bool = false ) -> [TokenGroup] {
+	if string.isEmpty {
+		return []
+	}
+	let maxCount = rule.openTag.count * rule.maxTags
+	var groups : [TokenGroup] = []
+	
+	let maxTag = rule.openTag.repeating(rule.maxTags)
+	
+	if maxTag.contains(string) {
+		if string.count == maxCount || shouldEmpty {
+			var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			token.state = .open
+			groups.append(token)
+			string.removeAll()
+		}
+	
+	} else if string == rule.intermediateTag {
+		var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		token.state = .intermediate
+		groups.append(token)
+		string.removeAll()
+	} else if string == rule.closingTag {
+		var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		token.state = .closed
+		groups.append(token)
+		string.removeAll()
+	}
+	
+	if shouldEmpty && !string.isEmpty {
+		let token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		groups.append(token)
+		string.removeAll()
+	}
+	return groups
+}
+
+func scan( _ string : String, with rule : Rule) -> [Token] {
+		let scanner = Scanner(string: string)
+		scanner.charactersToBeSkipped = nil
+		var tokens : [Token] = []
+		var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
+		if let existentEscape = rule.escapeCharacter {
+			set.insert(charactersIn: String(existentEscape))
+		}
+		
+		var openTag = rule.openTag.repeating(rule.maxTags)
+		
+		var tagString = TagString(with: rule)
+		
+		var openTagFound : TagState = .none
+		var regularCharacters = ""
+		var tagGroupCount = 0
+		while !scanner.isAtEnd {
+			tagGroupCount += 1
+			
+			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+				if let start = scanner.scanUpToCharacters(from: set) {
+					tagString.append(start)
+				}
+			} else {
+				var string : NSString?
+				scanner.scanUpToCharacters(from: set, into: &string)
+				if let existentString = string as String? {
+					tagString.append(existentString)
+				}
+			}
+			
+			// The end of the string
+			let maybeFoundChars = scanner.scanCharacters(from: set )
+			guard let foundTag = maybeFoundChars else {
+				continue
+			}
+			
+			if foundTag == rule.openTag && foundTag.count < rule.minTags {
+				tagString.append(foundTag)
+				continue
+			}
+			
+			//:--
+			print(foundTag)
+			var tokenGroups : [TokenGroup] = []
+			var escapeCharacter : Character? = nil
+			var cumulatedString = ""
+			for char in foundTag {
+				if let existentEscapeCharacter = escapeCharacter {
+					
+					// If any of the tags feature the current character
+					let escape = String(existentEscapeCharacter)
+					let nextTagCharacter = String(char)
+					if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
+						escapeCharacter = nil
+					} else if nextTagCharacter == escape {
+						// Doesn't apply to this rule
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
+					}
+					
+					continue
+				}
+				if let existentEscape = rule.escapeCharacter {
+					if char == existentEscape {
+						tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
+						escapeCharacter = char
+						continue
+					}
+				}
+				cumulatedString.append(char)
+				tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule))
+				
+			}
+			if let remainingEscape = escapeCharacter {
+				tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
+			}
+
+			tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
+			tagString.append(contentsOf: tokenGroups)
+			
+			if tagString.state == .closed {
+				tokens.append(contentsOf: tagString.tokens())
+			}
+			
+			
+		}
+		
+		tokens.append(contentsOf: tagString.tokens())
+		
+		
+		return tokens
+	}
+
+//: [Next](@next)
+
+
+
+var string = "[]([[\\[Some Link]\\]](\\(\\(\\url) [Regular link](url)"
+//string = "Text before [Regular link](url) Text after"
+var output = "[]([[Some Link]] Regular link"
+
+var tokens = scan(string, with: LinkRule())
+print( tokens.filter( { $0.type == .string }).map({ $0.outputString }).joined())
+//print( tokens )
+
+//string = "**\\*\\Bold\\*\\***"
+//output = "*\\Bold**"
+
+//tokens = scan(string, with: AsteriskRule())
+//print( tokens )
+

Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Sources/Tokens.swift

@@ -0,0 +1,32 @@
+import Foundation
+
+
+
+public protocol Rule {
+	var escapeCharacter : Character? { get }
+	var openTag : String { get }
+	var intermediateTag : String? { get }
+	var closingTag : String? { get }
+	var maxTags : Int { get }
+	var minTags : Int { get }
+}
+
+public struct LinkRule : Rule {
+	public let escapeCharacter : Character? = "\\"
+	public let openTag : String = "["
+	public let intermediateTag : String? = "]("
+	public let closingTag : String? = ")"
+	public let maxTags : Int = 1
+	public let minTags : Int = 1
+	public init() { }
+}
+
+public struct AsteriskRule : Rule {
+	public let escapeCharacter : Character? = "\\"
+	public let openTag : String = "*"
+	public let intermediateTag : String? = nil
+	public let closingTag : String? = nil
+	public let maxTags : Int = 3
+	public let minTags : Int = 1
+	public init() { }
+}

Playground/SwiftyMarkdown.playground/Sources/Swifty Tokeniser.swift

@@ -74,6 +74,7 @@ public struct Token {
 	public let inputString : String
 	public var metadataString : String? = nil
 	public var characterStyles : [CharacterStyling] = []
+	public var group : Int = 0
 	public var count : Int = 0
 	public var shouldSkip : Bool = false
 	public var outputString : String {

Playground/SwiftyMarkdown.playground/contents.xcplayground

@@ -5,5 +5,7 @@
         <page name='Line Processing'/>
         <page name='Tokenising'/>
         <page name='Attributed String'/>
+        <page name='SKLabelNode'/>
+        <page name='Groups'/>
     </pages>
 </playground>

Resources/test.md

@@ -62,3 +62,68 @@ Header 2
 	1. Including indented lists
 		- Up to three levels
 1. Neat! 
+
+# SwiftyMarkdown 1.0
+
+SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a Swift-style syntax. It uses dynamic type to set the font size correctly with whatever font you'd like to use.
+
+## Fully Rebuilt For 2020!
+
+SwiftyMarkdown now features a more robust and reliable rules-based line processing and tokenisation engine. It has added support for images stored in the bundle (`![Image](<Name In bundle>)`), codeblocks, blockquotes, and unordered lists!
+
+Line-level attributes can now have a paragraph alignment applied to them (e.g. `h2.aligment = .center`), and links can be underlined by setting underlineLinks to `true`. 
+
+It also uses the system color `.label` as the default font color on iOS 13 and above for Dark Mode support out of the box. 
+
+## Installation
+
+### CocoaPods:
+
+`pod 'SwiftyMarkdown'`
+
+### SPM: 
+
+In Xcode, `File -> Swift Packages -> Add Package Dependency` and add the GitHub URL. 
+
+*italics* or _italics_
+**bold** or __bold__
+~~Linethrough~~Strikethroughs. 
+`code`
+
+# Header 1
+
+or
+
+Header 1
+====
+
+## Header 2
+
+or
+
+Header 2
+---
+
+### Header 3
+#### Header 4
+##### Header 5 #####
+###### Header 6 ######
+
+	Indented code blocks (spaces or tabs)
+
+[Links](http://voyagetravelapps.com/)
+![Images](<Name of asset in bundle>)
+
+> Blockquotes
+
+- Bulleted
+- Lists
+	- Including indented lists
+		- Up to three levels
+- Neat!
+
+1. Ordered
+1. Lists
+	1. Including indented lists
+		- Up to three levels
+1. Neat! 

Sources/SwiftyMarkdown/SwiftyTokeniser.swift

@@ -44,6 +44,8 @@ public struct CharacterRule : CustomStringConvertible {
 	public var spacesAllowed : SpaceAllowed = .oneSide
 	public var cancels : Cancel = .none
 	
+	public var tagVarieties : [Int : String]
+	
 	public var description: String {
 		return "Character Rule with Open tag: \(self.openTag) and current styles : \(self.styles) "
 	}
@@ -57,6 +59,11 @@ public struct CharacterRule : CustomStringConvertible {
 		self.minTags = minTags
 		self.maxTags = maxTags
 		self.cancels = cancels
+		
+		self.tagVarieties = [:]
+		for i in minTags...maxTags {
+			self.tagVarieties[i] = openTag.repeating(i)
+		}
 	}
 }
 
@@ -77,6 +84,7 @@ public struct Token {
 	public let id = UUID().uuidString
 	public let type : TokenType
 	public let inputString : String
+	public fileprivate(set) var group : Int = 0
 	public fileprivate(set) var metadataString : String? = nil
 	public fileprivate(set) var characterStyles : [CharacterStyling] = []
 	public fileprivate(set) var count : Int = 0
@@ -84,6 +92,8 @@ public struct Token {
 	public fileprivate(set) var tokenIndex : Int = -1
 	public fileprivate(set) var isProcessed : Bool = false
 	public fileprivate(set) var isMetadata : Bool = false
+	
+	
 	public var outputString : String {
 		get {
 			switch self.type {
@@ -107,6 +117,9 @@ public struct Token {
 		self.type = type
 		self.inputString = inputString
 		self.characterStyles = characterStyles
+		if type == .repeatingTag {
+			self.count = inputString.count
+		}
 	}
 	
 	func newToken( fromSubstring string: String,  isReplacement : Bool) -> Token {
@@ -124,10 +137,215 @@ extension Sequence where Iterator.Element == Token {
 	}
 }
 
+enum TagState {
+	case none
+	case open
+	case intermediate
+	case closed
+}
+
+struct TagString {
+	var state : TagState = .none
+	var preOpenString = ""
+	var openTagString : [String] = []
+	var intermediateString = ""
+	var intermediateTagString = ""
+	var metadataString = ""
+	var closedTagString : [String] = []
+	var postClosedString = ""
+	
+	let rule : CharacterRule
+	var tokenGroup = 0
+	
+	init( with rule : CharacterRule ) {
+		self.rule = rule
+	}
+	
+	mutating func append( _ string : String? ) {
+		guard let existentString = string else {
+			return
+		}
+		switch self.state {
+		case .none:
+			self.preOpenString += existentString
+		case .open:
+			self.intermediateString += existentString
+		case .intermediate:
+			self.metadataString += existentString
+		case .closed:
+			self.postClosedString += existentString
+		}
+	}
+	
+	mutating func append( contentsOf tokenGroup: [TokenGroup] ) {
+		print(tokenGroup)
+		var availableCount = 0
+		for token in tokenGroup {
+			
+			switch token.state {
+			case .none:
+				self.append(token.string)
+				if self.state == .closed {
+					self.state = .none
+				}
+			case .open:
+				switch self.state {
+				case .none:
+					self.openTagString.append(token.string)
+					self.state = .open
+					availableCount = self.rule.maxTags - 1
+				case .open:
+					if self.rule.closingTag == nil {
+						if availableCount > 0 {
+							self.openTagString.append(token.string)
+							availableCount -= 1
+						} else {
+							self.closedTagString.append(token.string)
+							self.state = .closed
+						}
+					} else if self.rule.maxTags == 1, self.openTagString.first == rule.openTag {
+						self.preOpenString = self.preOpenString + self.openTagString.joined() + self.intermediateString
+						self.intermediateString = ""
+						self.openTagString.append(token.string)
+					} else {
+						self.openTagString.append(token.string)
+					}
+				case .intermediate:
+					self.preOpenString += self.openTagString.joined() + token.string
+				case .closed:
+					self.openTagString.append(token.string)
+				}
+			case .intermediate:
+				switch self.state {
+				case .none:
+					self.preOpenString += token.string
+				case .open:
+					self.intermediateTagString += token.string
+					self.state = .intermediate
+				case .intermediate:
+					self.metadataString += token.string
+				case .closed:
+					self.postClosedString += token.string
+				}
+				
+			case .closed:
+				switch self.state {
+				case .intermediate:
+					self.closedTagString.append(token.string)
+					self.state = .closed
+				case .closed:
+					self.postClosedString += token.string
+				case .open:
+					if self.rule.intermediateTag == nil {
+						self.closedTagString.append(token.string)
+						self.state = .closed
+					} else {
+						self.preOpenString += self.openTagString.joined()
+						self.preOpenString += self.intermediateString
+						self.preOpenString += token.string
+						self.intermediateString = ""
+						self.openTagString.removeAll()
+					}
+				case .none:
+					self.preOpenString += token.string
+				}
+			}
+		}
+		if !self.openTagString.isEmpty && self.rule.closingTag == nil && self.state != .closed {
+			self.state = .open
+		}
+	}
+	
+	func configureToken(ofType type : TokenType = .string, with string : String ) -> Token {
+		var token = Token(type: type, inputString: string)
+		token.group = self.tokenGroup
+		return token
+	}
+	
+	mutating func reset() {
+		self.preOpenString = ""
+		self.openTagString.removeAll()
+		self.intermediateString = ""
+		self.intermediateTagString = ""
+		self.metadataString = ""
+		self.closedTagString.removeAll()
+		self.postClosedString = ""
+		
+		self.state = .none
+	}
+	
+	mutating func tokens(beginningGroupNumberAt group : Int = 0) -> [Token] {
+		print(self)
+		self.tokenGroup = group
+		var tokens : [Token] = []
+		
+		if self.intermediateString.isEmpty && self.intermediateTagString.isEmpty && self.metadataString.isEmpty {
+			tokens.append(self.configureToken(with: self.preOpenString + self.openTagString.joined() + self.closedTagString.joined() + self.postClosedString))
+			self.reset()
+			return tokens
+		}
+		
+		if !self.preOpenString.isEmpty {
+			tokens.append(self.configureToken(with: self.preOpenString))
+		}
+		
+		for tag in self.openTagString {
+			if self.rule.closingTag == nil {
+				tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
+			} else {
+				tokens.append(self.configureToken(ofType: .openTag, with: tag))
+			}
+		}
+		self.tokenGroup += 1
+		if !self.intermediateString.isEmpty {
+			var token = self.configureToken(with: self.intermediateString)
+			token.metadataString = self.metadataString
+			tokens.append(token)
+		}
+		if !self.intermediateTagString.isEmpty {
+			tokens.append(self.configureToken(ofType: .intermediateTag, with: self.intermediateTagString))
+		}
+		
+		self.tokenGroup += 1
+		
+		if !self.metadataString.isEmpty {
+			tokens.append(self.configureToken(with: self.metadataString))
+		}
+		for tag in self.closedTagString {
+			if self.rule.closingTag == nil {
+				tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
+			} else {
+				tokens.append(self.configureToken(ofType: .closeTag, with: tag))
+			}
+		}
+		if !self.postClosedString.isEmpty {
+			tokens.append(self.configureToken(with: self.postClosedString))
+		}
+		
+		self.reset()
+		
+		return tokens
+	}
+}
+
+struct TokenGroup {
+	enum TokenGroupType {
+		case string
+		case tag
+		case escape
+	}
+	
+	let string : String
+	let isEscaped : Bool
+	let type : TokenGroupType
+	var state : TagState = .none
+}
+
 public class SwiftyTokeniser {
 	let rules : [CharacterRule]
 	var replacements : [String : [Token]] = [:]
 	
+	var timer : TimeInterval = 0
 	var enableLog = (ProcessInfo.processInfo.environment["SwiftyTokeniserLogging"] != nil)
 	
 	public init( with rules : [CharacterRule] ) {
@@ -154,7 +372,8 @@ public class SwiftyTokeniser {
 		var currentTokens : [Token] = []
 		var mutableRules = self.rules
 		
-		
+		self.timer = Date().timeIntervalSinceReferenceDate
+		//		os_log("TIMER BEGIN: 0", log: .tokenising, type: .info)
 		
 		while !mutableRules.isEmpty {
 			let nextRule = mutableRules.removeFirst()
@@ -487,7 +706,7 @@ public class SwiftyTokeniser {
 		
 		let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
 		// Try to find exact match first
-		if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed }) {
+		if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) {
 			endIdx = nextTokenIdx
 		}
 		
@@ -594,45 +813,14 @@ public class SwiftyTokeniser {
 		return mutableTokens
 	}
 	
-	enum TagState {
-		case open
-		case intermediate
-		case closed
-	}
-	
-	func scan( _ string : String, with rule : CharacterRule) -> [Token] {
-		let scanner = Scanner(string: string)
-		scanner.charactersToBeSkipped = nil
-		var tokens : [Token] = []
-		var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
-		if let existentEscape = rule.escapeCharacter {
-			set.insert(charactersIn: String(existentEscape))
-		}
-		
-		
-		var openTagFound : TagState = .open
-		var openingString = ""
-		while !scanner.isAtEnd {
-			
-			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				if let start = scanner.scanUpToCharacters(from: set) {
-					openingString.append(start)
-				}
-			} else {
-				var string : NSString?
-				scanner.scanUpToCharacters(from: set, into: &string)
-				if let existentString = string as String? {
-					openingString.append(existentString)
-				}
-				// Fallback on earlier versions
-			}
 	
+	func scanSpacing( _ scanner : Scanner, usingCharactersIn set : CharacterSet ) -> (preTag : String?, foundChars : String?, postTag : String?) {
 		let lastChar : String?
 		if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				lastChar = ( scanner.currentIndex > string.startIndex ) ? String(string[string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
+			lastChar = ( scanner.currentIndex > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
 		} else {
-				if let scanLocation = string.index(string.startIndex, offsetBy: scanner.scanLocation, limitedBy: string.endIndex) {
+			if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
-					lastChar = ( scanLocation > string.startIndex ) ? String(string[string.index(before: scanLocation)..<scanLocation]) : nil
+				lastChar = ( scanLocation > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanLocation)..<scanLocation]) : nil
 			} else {
 				lastChar = nil
 			}
@@ -649,153 +837,172 @@ public class SwiftyTokeniser {
 		
 		let nextChar : String?
 		if #available(iOS 13.0, OSX 10.15,  watchOS 6.0,tvOS 13.0, *) {
-				 nextChar = (scanner.currentIndex != string.endIndex) ? String(string[scanner.currentIndex]) : nil
+			nextChar = (scanner.currentIndex != scanner.string.endIndex) ? String(scanner.string[scanner.currentIndex]) : nil
 		} else {
-				if let scanLocation = string.index(string.startIndex, offsetBy: scanner.scanLocation, limitedBy: string.endIndex) {
+			if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
-					nextChar = (scanLocation != string.endIndex) ? String(string[scanLocation]) : nil
+				nextChar = (scanLocation != scanner.string.endIndex) ? String(scanner.string[scanLocation]) : nil
 			} else {
 				nextChar = nil
 			}
-				
+		}
+		return (lastChar, maybeFoundChars, nextChar)
 	}
 	
-			guard let foundChars = maybeFoundChars else {
+	func getTokenGroups( for string : inout String, with rule : CharacterRule, shouldEmpty : Bool = false ) -> [TokenGroup] {
-				tokens.append(Token(type: .string, inputString: "\(openingString)"))
+		if string.isEmpty {
-				openingString = ""
+			return []
+		}
+		var groups : [TokenGroup] 	= []
+		
+		if string.contains(rule.openTag) {
+			if shouldEmpty || string == rule.tagVarieties[rule.maxTags]{
+				var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+				token.state = .open
+				groups.append(token)
+				string.removeAll()
+			}
+			
+		} else if let intermediateString = rule.intermediateTag, string.contains(intermediateString)  {
+			
+			if let range = string.range(of: intermediateString) {
+				let prior = string[string.startIndex..<range.lowerBound]
+				let tag = string[range]
+				let following = string[range.upperBound..<string.endIndex]
+				if !prior.isEmpty {
+					groups.append(TokenGroup(string: String(prior), isEscaped: false, type: .string))
+				}
+				var token = TokenGroup(string: String(tag), isEscaped: false, type: .tag)
+				token.state = .intermediate
+				groups.append(token)
+				if !following.isEmpty {
+					groups.append(TokenGroup(string: String(following), isEscaped: false, type: .string))
+				}
+				string.removeAll()
+			}
+		} else if let closingTag = rule.closingTag, closingTag.contains(string) {
+			var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			token.state = .closed
+			groups.append(token)
+			string.removeAll()
+		}
+		
+		if shouldEmpty && !string.isEmpty {
+			let token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			groups.append(token)
+			string.removeAll()
+		}
+		return groups
+	}
+	
+	func scan( _ string : String, with rule : CharacterRule) -> [Token] {
+		os_log("TIMER CHECK: %f for rule with openTag %@", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg, rule.openTag)
+		let scanner = Scanner(string: string)
+		scanner.charactersToBeSkipped = nil
+		var tokens : [Token] = []
+		var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
+		if let existentEscape = rule.escapeCharacter {
+			set.insert(charactersIn: String(existentEscape))
+		}
+		
+		
+		var tagString = TagString(with: rule)
+		var tokenGroup = 0
+		while !scanner.isAtEnd {
+			tokenGroup += 1
+			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+				if let start = scanner.scanUpToCharacters(from: set) {
+					tagString.append(start)
+				}
+			} else {
+				var string : NSString?
+				scanner.scanUpToCharacters(from: set, into: &string)
+				if let existentString = string as String? {
+					tagString.append(existentString)
+				}
+			}
+			
+			//				os_log("TIMER CHECK (pre-spacing): %f", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg)
+			
+			// The end of the string
+			let spacing = self.scanSpacing(scanner, usingCharactersIn: set)
+			
+			
+			guard let foundTag = spacing.foundChars else {
 				continue
 			}
 			
-			if foundChars == rule.openTag && foundChars.count < rule.minTags {
+			if foundTag == rule.openTag && foundTag.count < rule.minTags {
-				openingString.append(foundChars)
+				tagString.append(foundTag)
 				continue
 			}
 			
-			if !validateSpacing(nextCharacter: nextChar, previousCharacter: lastChar, with: rule) {
+			if !validateSpacing(nextCharacter: spacing.postTag, previousCharacter: spacing.preTag, with: rule) {
 				let escapeString = String("\(rule.escapeCharacter ?? Character(""))")
-				var escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
+				var escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
 				if let hasIntermediateTag = rule.intermediateTag {
-					escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
+					escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
 				}
 				if let existentClosingTag = rule.closingTag {
-					escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
+					escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
 				}
-				
+				tagString.append(escaped)
-				openingString.append(escaped)
 				continue
 			}
 			
-			// Here's where we have to do the actual tag management.
+			//				os_log("TIMER CHECK (pre grouping): %f", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg)
+			//:--
+			print(foundTag)
 			
-			var cumulativeString = ""
+			if !foundTag.contains(rule.openTag) && !foundTag.contains(rule.intermediateTag ?? "") && !foundTag.contains(rule.closingTag ?? "") {
-			var openString = ""
+				tagString.append(foundTag)
-			var containedText = ""
+				continue
-			var intermediateString = ""
-			var metadataText = ""
-			var closedString = ""
-			var maybeEscapeNext = false
-			
-			
-			func addToken( for type : TokenType ) {
-				var inputString : String
-				switch type {
-				case .openTag:
-					inputString = openString
-				case .intermediateTag:
-					inputString = intermediateString
-				case .closeTag:
-					inputString = closedString
-				default:
-					inputString = ""
-				}
-				guard !inputString.isEmpty else {
-					return
-				}
-				guard inputString.count >= rule.minTags else {
-					return
 			}
 			
-				if !openingString.isEmpty {
-					tokens.append(Token(type: .string, inputString: "\(openingString)"))
-					openingString = ""
-				}
-				let actualType : TokenType = ( rule.intermediateTag == nil && rule.closingTag == nil ) ? .repeatingTag : type
 			
-				var token = Token(type: actualType, inputString: inputString)
+			var tokenGroups : [TokenGroup] = []
-				if rule.closingTag == nil {
+			var escapeCharacter : Character? = nil
-					token.count = inputString.count
+			var cumulatedString = ""
+			for char in foundTag {
+				if let existentEscapeCharacter = escapeCharacter {
+					
+					// If any of the tags feature the current character
+					let escape = String(existentEscapeCharacter)
+					let nextTagCharacter = String(char)
+					if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
+						escapeCharacter = nil
+					} else if nextTagCharacter == escape {
+						// Doesn't apply to this rule
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
 					}
 					
-				tokens.append(token)
+					continue
-				
-				switch type {
-				case .openTag:
-					openString = ""
-				case .intermediateTag:
-					intermediateString = ""
-				case .closeTag:
-					closedString = ""
-				default:
-					break
-				}
-			}
-			
-			// Here I am going through and adding the characters in the found set to a cumulative string.
-			// If there is an escape character, then the loop stops and any open tags are tokenised.
-			for char in foundChars {
-				cumulativeString.append(char)
-				if maybeEscapeNext {
-					
-					var escaped = cumulativeString
-					if String(char) == rule.openTag || String(char) == rule.intermediateTag || String(char) == rule.closingTag {
-						escaped = String(cumulativeString.replacingOccurrences(of: String(rule.escapeCharacter ?? Character("")), with: ""))
-					}
-					
-					openingString.append(escaped)
-					cumulativeString = ""
-					maybeEscapeNext = false
 				}
 				if let existentEscape = rule.escapeCharacter {
-					if cumulativeString == String(existentEscape) {
+					if char == existentEscape {
-						maybeEscapeNext = true
+						tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
-						addToken(for: .openTag)
+						escapeCharacter = char
-						addToken(for: .intermediateTag)
-						addToken(for: .closeTag)
 						continue
 					}
 				}
+				cumulatedString.append(char)
+				tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule))
 				
+			}
+			if let remainingEscape = escapeCharacter {
+				tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
+			}
 			
-				if cumulativeString == rule.openTag, openTagFound == .open {
+			tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
-					openString.append(char)
+			tagString.append(contentsOf: tokenGroups)
-					cumulativeString = ""
+			
-					openTagFound = ( rule.closingTag == nil ) ? .open : .closed
+			if tagString.state == .none {
-					openTagFound = ( rule.intermediateTag == nil ) ? openTagFound : .intermediate
+				tokens.append(contentsOf: tagString.tokens(beginningGroupNumberAt : tokenGroup))
-				} else if cumulativeString == rule.intermediateTag, openTagFound == .intermediate {
-					intermediateString.append(cumulativeString)
-					cumulativeString = ""
-					openTagFound = ( rule.closingTag == nil ) ? .open : .closed
-				} else if cumulativeString == rule.closingTag, openTagFound == .closed {
-					closedString.append(char)
-					cumulativeString = ""
-					openTagFound = .open
 			}
 		}
 		
-			
+		tokens.append(contentsOf: tagString.tokens(beginningGroupNumberAt : tokenGroup))
-			addToken(for: .openTag)
+		//			os_log("TIMER TOTAL: %f for rule with openTag %@", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg, rule.openTag)
-			addToken(for: .intermediateTag)
-			addToken(for: .closeTag)
-			openingString.append( cumulativeString )
-			
-			// If we're here, it means that an escape character was found but without a corresponding
-			// tag, which means it might belong to a different rule.
-			// It should be added to the next group of regular characters
-
-		}
-		
-		if !openingString.isEmpty {
-			tokens.append(Token(type: .string, inputString: "\(openingString)"))
-		}
 		
 		return tokens
 	}
@@ -838,3 +1045,14 @@ public class SwiftyTokeniser {
 	}
 	
 }
+
+
+extension String {
+	func repeating( _ max : Int ) -> String {
+		var output = self
+		for _ in 1..<max {
+			output += self
+		}
+		return output
+	}
+}

SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/AD1DF83E-20BC-4E7E-8C14-683818ED0A26.plist

@@ -11,9 +11,11 @@
 				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
 				<dict>
 					<key>baselineAverage</key>
-					<real>0.0217</real>
+					<real>0.01</real>
 					<key>baselineIntegrationDisplayName</key>
 					<string>Local Baseline</string>
+					<key>maxPercentRelativeStandardDeviation</key>
+					<real>10</real>
 				</dict>
 			</dict>
 			<key>testThatStringsAreProcessedQuickly()</key>

Tests/SwiftyMarkdownTests/SwiftyMarkdownCharacterTests.swift

@@ -12,36 +12,50 @@ import XCTest
 class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	
 	func testIsolatedCase() {
-		let challenge = TokenTest(input: "An *\\*italic\\** [referenced link][link]", output: "An *italic* referenced link", tokens: [
+
+		challenge = TokenTest(input: "A string with a **bold** word", output: "A string with a bold word",  tokens: [
+			Token(type: .string, inputString: "A string with a ", characterStyles: []),
+			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
+			Token(type: .string, inputString: " word", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		return
+		
+		challenge = TokenTest(input: "An *\\*italic\\** [referenced link][link]", output: "An *italic* referenced link", tokens: [
 			Token(type: .string, inputString: "An ", characterStyles: []),
 			Token(type: .string, inputString: "*italic*", characterStyles: [CharacterStyle.italic]),
 			Token(type: .string, inputString: " ", characterStyles: []),
 			Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])
 		])
-		let rules : [CharacterRule] = [
+		rules  = [
 			CharacterRule(openTag: "*", escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [.bold], 3 : [.italic, .bold]], minTags: 1, maxTags: 3),
 			CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], minTags: 1, maxTags: 1),
 			CharacterRule(openTag: "[", intermediateTag: "][", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], minTags: 1, maxTags: 1)
 		]
-		let results = self.attempt(challenge, rules: rules)
+		results = self.attempt(challenge, rules: rules)
 		XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
-		var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
 		
-		if links.count == 1 {
+		
-			XCTAssertEqual(links[0].metadataString, "https://www.neverendingvoyage.com/")
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataString, "https://www.neverendingvoyage.com/")
 		} else {
-			XCTFail("Incorrect link count. Expecting 1, found \(links.count)")
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
 		}
 		
 	}
 	
 	func testThatBoldTraitsAreRecognised() {
-		var challenge = TokenTest(input: "**A bold string**", output: "A bold string",  tokens: [
+		challenge = TokenTest(input: "**A bold string**", output: "A bold string",  tokens: [
 			Token(type: .string, inputString: "A bold string", characterStyles: [CharacterStyle.bold])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -67,6 +81,15 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
+		challenge = TokenTest(input: "\\\\*\\*A normal \\\\ string\\*\\*", output: "\\**A normal \\\\ string**", tokens: [
+			Token(type: .string, inputString: "\\**A normal \\\\ string**", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
 		challenge = TokenTest(input: "A string with double \\*\\*escaped\\*\\* asterisks", output: "A string with double **escaped** asterisks", tokens: [
 			Token(type: .string, inputString: "A string with double **escaped** asterisks", characterStyles: [])
 		])
@@ -99,10 +122,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	}
 	
 	func testThatCodeTraitsAreRecognised() {
-		var challenge = TokenTest(input: "`Code (**should** not process internal tags)`", output: "Code (**should** not process internal tags)",  tokens: [
+		challenge = TokenTest(input: "`Code (**should** not process internal tags)`", output: "Code (**should** not process internal tags)",  tokens: [
 			Token(type: .string, inputString: "Code (**should** not process internal tags) ", characterStyles: [CharacterStyle.code])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -158,14 +181,23 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "Two backticks followed by a full stop ``.", output: "Two backticks followed by a full stop ``.", tokens: [
+			Token(type: .string, inputString: "Two backticks followed by a full stop ``.", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
 	}
 	
 	func testThatItalicTraitsAreParsedCorrectly() {
 		
-		var challenge = TokenTest(input: "*An italicised string*", output: "An italicised string", tokens : [
+		challenge = TokenTest(input: "*An italicised string*", output: "An italicised string", tokens : [
 			Token(type: .string, inputString: "An italicised string", characterStyles: [CharacterStyle.italic])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -236,11 +268,11 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	}
 	
 	func testThatStrikethroughTraitsAreRecognised() {
-		var challenge = TokenTest(input: "~~An~~A crossed-out string", output: "AnA crossed-out string", tokens: [
+		challenge = TokenTest(input: "~~An~~A crossed-out string", output: "AnA crossed-out string", tokens: [
 			Token(type: .string, inputString: "An", characterStyles: [CharacterStyle.strikethrough]),
 			Token(type: .string, inputString: "A crossed-out string", characterStyles: [])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -269,7 +301,7 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	
 	func testThatMixedTraitsAreRecognised() {
 		
-		var challenge = TokenTest(input: "__A bold string__ with a **mix** **of** bold __styles__", output: "A bold string with a mix of bold styles", tokens : [
+		challenge = TokenTest(input: "__A bold string__ with a **mix** **of** bold __styles__", output: "A bold string with a mix of bold styles", tokens : [
 			Token(type: .string, inputString: "A bold string", characterStyles: [CharacterStyle.bold]),
 			Token(type: .string, inputString: "with a ", characterStyles: []),
 			Token(type: .string, inputString: "mix", characterStyles: [CharacterStyle.bold]),
@@ -278,7 +310,7 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 			Token(type: .string, inputString: " bold ", characterStyles: []),
 			Token(type: .string, inputString: "styles", characterStyles: [CharacterStyle.bold])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -306,10 +338,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	}
 	
 	func testThatExtraCharactersAreHandles() {
-		var challenge = TokenTest(input: "***A bold italic string***", output: "A bold italic string",  tokens: [
+		challenge = TokenTest(input: "***A bold italic string***", output: "A bold italic string",  tokens: [
 			Token(type: .string, inputString: "A bold italic string", characterStyles: [CharacterStyle.bold, CharacterStyle.italic])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -380,12 +412,12 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	
 	func offtestAdvancedEscaping() {
 		
-		var challenge = TokenTest(input: "\\***A normal string*\\**", output: "**A normal string*", tokens: [
+		challenge = TokenTest(input: "\\***A normal string*\\**", output: "**A normal string*", tokens: [
 			Token(type: .string, inputString: "**", characterStyles: []),
 			Token(type: .string, inputString: "A normal string", characterStyles: [CharacterStyle.italic]),
 			Token(type: .string, inputString: "**", characterStyles: [])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -411,7 +443,6 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 		let asteriskComma = "An asterisk followed by a full stop: *, *"
 		
 		let backtickSpace = "A backtick followed by a space: `"
-		let backtickFullStop = "Two backticks followed by a full stop: ``."
 		
 		let underscoreSpace = "An underscore followed by a space: _"
 
@@ -431,9 +462,6 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 		md = SwiftyMarkdown(string: asteriskFullStop)
 		XCTAssertEqual(md.attributedString().string, asteriskFullStop)
 		
-		md = SwiftyMarkdown(string: backtickFullStop)
-		XCTAssertEqual(md.attributedString().string, backtickFullStop)
-		
 		md = SwiftyMarkdown(string: underscoreFullStop)
 		XCTAssertEqual(md.attributedString().string, underscoreFullStop)
 		
@@ -458,10 +486,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	}
 	
 	func testReportedCrashingStrings() {
-		let challenge = TokenTest(input: "[**\\!bang**](https://duckduckgo.com/bang)", output: "\\!bang", tokens: [
+		challenge = TokenTest(input: "[**\\!bang**](https://duckduckgo.com/bang)", output: "\\!bang", tokens: [
 			Token(type: .string, inputString: "\\!bang", characterStyles: [CharacterStyle.bold, CharacterStyle.link])
 		])
-		let results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)

Tests/SwiftyMarkdownTests/SwiftyMarkdownLinkTests.swift

@@ -13,10 +13,10 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 	
 	func testForLinks() {
 		
-		var challenge = TokenTest(input: "[Link at start](http://voyagetravelapps.com/)", output: "Link at start", tokens: [
+		challenge = TokenTest(input: "[Link at start](http://voyagetravelapps.com/)", output: "Link at start", tokens: [
 			Token(type: .string, inputString: "Link at start", characterStyles: [CharacterStyle.link])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -82,25 +82,10 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 		}
 
 	
-		challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/)", tokens: [
-			Token(type: .string, inputString: "Link with missing square(http://voyagetravelapps.com/)", characterStyles: [])
-		])
-		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
-		XCTAssertEqual(results.foundStyles, results.expectedStyles)
-		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
-		challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/)", tokens: [
+	}
-			Token(type: .string, inputString: "A ", characterStyles: []),
-			Token(type: .string, inputString: "[Link(http://voyagetravelapps.com/)", characterStyles: [])
-		])
-		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
-		XCTAssertEqual(results.foundStyles, results.expectedStyles)
-		XCTAssertEqual(results.attributedString.string, challenge.output)
 	
+	func testMalformedLinks() {
 		
 		challenge = TokenTest(input: "[Link with missing parenthesis](http://voyagetravelapps.com/", output: "[Link with missing parenthesis](http://voyagetravelapps.com/", tokens: [
 			Token(type: .string, inputString: "[Link with missing parenthesis](", characterStyles: []),
@@ -123,12 +108,54 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
-		challenge = TokenTest(input: "[Link1](http://voyagetravelapps.com/) **bold** [Link2](http://voyagetravelapps.com/)", output: "Link1 bold Link2",  tokens: [
+		challenge = TokenTest(input: "[A link](((url)", output: "A link", tokens: [
-			Token(type: .string, inputString: "Link1", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: "A link", characterStyles: [CharacterStyle.link])
-			Token(type: .string, inputString: " ", characterStyles: []),
+		])
-			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
+		rules = [
-			Token(type: .string, inputString: " ", characterStyles: []),
+			CharacterRule(openTag: "![", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.image]]),
-			Token(type: .string, inputString: "Link2", characterStyles: [CharacterStyle.link])
+			CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]]),
+		]
+		results = self.attempt(challenge, rules: rules)
+		
+		
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataString, "((url")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/)", tokens: [
+			Token(type: .string, inputString: "Link with missing square(http://voyagetravelapps.com/)", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "[Link with [second opening](http://voyagetravelapps.com/)", output: "[Link with second opening", tokens: [
+			Token(type: .string, inputString: "Link with ", characterStyles: []),
+			Token(type: .string, inputString: "second opening", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataString, "http://voyagetravelapps.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/)", tokens: [
+			Token(type: .string, inputString: "A [Link(http://voyagetravelapps.com/)", characterStyles: [])
 		])
 		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
@@ -173,6 +200,19 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 		} else {
 			XCTFail("Incorrect link count. Expecting 1, found \(links.count)")
 		}
+		
+		challenge = TokenTest(input: "[Link1](http://voyagetravelapps.com/) **bold** [Link2](http://voyagetravelapps.com/)", output: "Link1 bold Link2",  tokens: [
+			Token(type: .string, inputString: "Link1", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: " ", characterStyles: []),
+			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
+			Token(type: .string, inputString: " ", characterStyles: []),
+			Token(type: .string, inputString: "Link2", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
 	}
 	
 	func testForImages() {

Tests/SwiftyMarkdownTests/XCTest+SwiftyMarkdown.swift

@@ -9,11 +9,41 @@
 import XCTest
 @testable import SwiftyMarkdown
 
+
+struct ChallengeReturn {
+	let tokens : [Token]
+	let stringTokens : [Token]
+	let links : [Token]
+	let attributedString : NSAttributedString
+	let foundStyles : [[CharacterStyle]]
+	let expectedStyles : [[CharacterStyle]]
+}
+
 class SwiftyMarkdownCharacterTests : XCTestCase {
 	let defaultRules = SwiftyMarkdown.characterRules
 	
-	func testDummy() {
+	var challenge : TokenTest!
+	var results : ChallengeReturn!
+	var rules : [CharacterRule]? = nil
 	
+	func attempt( _ challenge : TokenTest, rules : [CharacterRule]? = nil ) -> ChallengeReturn {
+		if let validRules = rules {
+			SwiftyMarkdown.characterRules = validRules
+		} else {
+			SwiftyMarkdown.characterRules = self.defaultRules
+		}
+		
+		let md = SwiftyMarkdown(string: challenge.input)
+		let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
+		let tokens = tokeniser.process(challenge.input)
+		let stringTokens = tokens.filter({ $0.type == .string && !$0.isMetadata })
+		
+		let existentTokenStyles = stringTokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
+		let expectedStyles = challenge.tokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
+		
+		let linkTokens = tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		
+		return ChallengeReturn(tokens: tokens, stringTokens: stringTokens, links : linkTokens, attributedString:  md.attributedString(), foundStyles: existentTokenStyles, expectedStyles : expectedStyles)
 	}
 }
 
@@ -29,22 +59,4 @@ extension XCTestCase {
 
 }
 
-extension SwiftyMarkdownCharacterTests {
-	func attempt( _ challenge : TokenTest, rules : [CharacterRule]? = nil ) -> (tokens : [Token], stringTokens: [Token], attributedString : NSAttributedString, foundStyles : [[CharacterStyle]], expectedStyles : [[CharacterStyle]] ) {
-		if let validRules = rules {
-			SwiftyMarkdown.characterRules = validRules
-		} else {
-			SwiftyMarkdown.characterRules = self.defaultRules
-		}
 
-		let md = SwiftyMarkdown(string: challenge.input)
-		let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
-		let tokens = tokeniser.process(challenge.input)
-		let stringTokens = tokens.filter({ $0.type == .string && !$0.isMetadata })
-		
-		let existentTokenStyles = stringTokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
-		let expectedStyles = challenge.tokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
-		
-		return (tokens, stringTokens, md.attributedString(), existentTokenStyles, expectedStyles)
-	}
-}