Completely overhauls tokenising engine

Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Contents.swift

@@ -0,0 +1,285 @@
+//: [Previous](@previous)
+
+import Foundation
+
+extension String {
+	func repeating( _ max : Int ) -> String {
+		var output = self
+		for _ in 1..<max {
+			output += self
+		}
+		return output
+	}
+}
+
+enum TagState {
+	case none
+	case open
+	case intermediate
+	case closed
+}
+
+struct TagString {
+	var state : TagState = .none
+	var preOpenString = ""
+	var openTagString = ""
+	var intermediateString = ""
+	var intermediateTagString = ""
+	var metadataString = ""
+	var closedTagString = ""
+	var postClosedString = ""
+
+	let rule : Rule
+	
+	init( with rule : Rule ) {
+		self.rule = rule
+	}
+	
+	mutating func append( _ string : String? ) {
+		guard let existentString = string else {
+			return
+		}
+		switch self.state {
+		case .none:
+			self.preOpenString += existentString
+		case .open:
+			self.intermediateString += existentString
+		case .intermediate:
+			self.metadataString += existentString
+		case .closed:
+			self.postClosedString += existentString
+		}
+	}
+	
+	mutating func append( contentsOf tokenGroup: [TokenGroup] ) {
+		print(tokenGroup)
+		for token in tokenGroup {
+			switch token.state {
+			case .none:
+				self.append(token.string)
+			case .open:
+				if self.state != .none {
+					self.preOpenString += token.string
+				} else {
+					self.openTagString += token.string
+				}
+			case .intermediate:
+				if self.state != .open {
+					self.intermediateString += token.string
+				} else {
+					self.intermediateTagString += token.string
+				}
+			case .closed:
+				if self.rule.intermediateTag != nil && self.state != .intermediate {
+					self.metadataString += token.string
+				} else {
+					self.closedTagString += token.string
+				}
+			}
+			self.state = token.state
+		}
+	}
+	
+	mutating func tokens() -> [Token] {
+		print(self)
+		var tokens : [Token] = []
+
+		if !self.preOpenString.isEmpty {
+			tokens.append(Token(type: .string, inputString: self.preOpenString))
+		}
+		if !self.openTagString.isEmpty {
+			tokens.append(Token(type: .openTag, inputString: self.openTagString))
+		}
+		if !self.intermediateString.isEmpty {
+			var token = Token(type: .string, inputString: self.intermediateString)
+			token.metadataString = self.metadataString
+			tokens.append(token)
+		}
+		if !self.intermediateTagString.isEmpty {
+			tokens.append(Token(type: .intermediateTag, inputString: self.intermediateTagString))
+		}
+		if !self.metadataString.isEmpty {
+			tokens.append(Token(type: .metadata, inputString: self.metadataString))
+		}
+		if !self.closedTagString.isEmpty {
+			tokens.append(Token(type: .closeTag, inputString: self.closedTagString))
+		}
+		
+		self.preOpenString = ""
+		self.openTagString = ""
+		self.intermediateString = ""
+		self.intermediateTagString = ""
+		self.metadataString = ""
+		self.closedTagString = ""
+		self.postClosedString = ""
+		
+		self.state = .none
+		
+		return tokens
+	}
+}
+
+struct TokenGroup {
+	enum TokenGroupType {
+		case string
+		case tag
+		case escape
+	}
+
+	let string : String
+	let isEscaped : Bool
+	let type : TokenGroupType
+	var state : TagState = .none
+}
+
+
+
+func getTokenGroups( for string : inout String, with rule : Rule, shouldEmpty : Bool = false ) -> [TokenGroup] {
+	if string.isEmpty {
+		return []
+	}
+	let maxCount = rule.openTag.count * rule.maxTags
+	var groups : [TokenGroup] = []
+	
+	let maxTag = rule.openTag.repeating(rule.maxTags)
+	
+	if maxTag.contains(string) {
+		if string.count == maxCount || shouldEmpty {
+			var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			token.state = .open
+			groups.append(token)
+			string.removeAll()
+		}
+	
+	} else if string == rule.intermediateTag {
+		var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		token.state = .intermediate
+		groups.append(token)
+		string.removeAll()
+	} else if string == rule.closingTag {
+		var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		token.state = .closed
+		groups.append(token)
+		string.removeAll()
+	}
+	
+	if shouldEmpty && !string.isEmpty {
+		let token = TokenGroup(string: string, isEscaped: false, type: .tag)
+		groups.append(token)
+		string.removeAll()
+	}
+	return groups
+}
+
+func scan( _ string : String, with rule : Rule) -> [Token] {
+		let scanner = Scanner(string: string)
+		scanner.charactersToBeSkipped = nil
+		var tokens : [Token] = []
+		var set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")")
+		if let existentEscape = rule.escapeCharacter {
+			set.insert(charactersIn: String(existentEscape))
+		}
+		
+		var openTag = rule.openTag.repeating(rule.maxTags)
+		
+		var tagString = TagString(with: rule)
+		
+		var openTagFound : TagState = .none
+		var regularCharacters = ""
+		var tagGroupCount = 0
+		while !scanner.isAtEnd {
+			tagGroupCount += 1
+			
+			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+				if let start = scanner.scanUpToCharacters(from: set) {
+					tagString.append(start)
+				}
+			} else {
+				var string : NSString?
+				scanner.scanUpToCharacters(from: set, into: &string)
+				if let existentString = string as String? {
+					tagString.append(existentString)
+				}
+			}
+			
+			// The end of the string
+			let maybeFoundChars = scanner.scanCharacters(from: set )
+			guard let foundTag = maybeFoundChars else {
+				continue
+			}
+			
+			if foundTag == rule.openTag && foundTag.count < rule.minTags {
+				tagString.append(foundTag)
+				continue
+			}
+			
+			//:--
+			print(foundTag)
+			var tokenGroups : [TokenGroup] = []
+			var escapeCharacter : Character? = nil
+			var cumulatedString = ""
+			for char in foundTag {
+				if let existentEscapeCharacter = escapeCharacter {
+					
+					// If any of the tags feature the current character
+					let escape = String(existentEscapeCharacter)
+					let nextTagCharacter = String(char)
+					if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
+						escapeCharacter = nil
+					} else if nextTagCharacter == escape {
+						// Doesn't apply to this rule
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
+					}
+					
+					continue
+				}
+				if let existentEscape = rule.escapeCharacter {
+					if char == existentEscape {
+						tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
+						escapeCharacter = char
+						continue
+					}
+				}
+				cumulatedString.append(char)
+				tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule))
+				
+			}
+			if let remainingEscape = escapeCharacter {
+				tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
+			}
+
+			tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
+			tagString.append(contentsOf: tokenGroups)
+			
+			if tagString.state == .closed {
+				tokens.append(contentsOf: tagString.tokens())
+			}
+			
+			
+		}
+		
+		tokens.append(contentsOf: tagString.tokens())
+		
+		
+		return tokens
+	}
+
+//: [Next](@next)
+
+
+
+var string = "[]([[\\[Some Link]\\]](\\(\\(\\url) [Regular link](url)"
+//string = "Text before [Regular link](url) Text after"
+var output = "[]([[Some Link]] Regular link"
+
+var tokens = scan(string, with: LinkRule())
+print( tokens.filter( { $0.type == .string }).map({ $0.outputString }).joined())
+//print( tokens )
+
+//string = "**\\*\\Bold\\*\\***"
+//output = "*\\Bold**"
+
+//tokens = scan(string, with: AsteriskRule())
+//print( tokens )
+

Playground/SwiftyMarkdown.playground/Pages/Groups.xcplaygroundpage/Sources/Tokens.swift

@@ -0,0 +1,32 @@
+import Foundation
+
+
+
+public protocol Rule {
+	var escapeCharacter : Character? { get }
+	var openTag : String { get }
+	var intermediateTag : String? { get }
+	var closingTag : String? { get }
+	var maxTags : Int { get }
+	var minTags : Int { get }
+}
+
+public struct LinkRule : Rule {
+	public let escapeCharacter : Character? = "\\"
+	public let openTag : String = "["
+	public let intermediateTag : String? = "]("
+	public let closingTag : String? = ")"
+	public let maxTags : Int = 1
+	public let minTags : Int = 1
+	public init() { }
+}
+
+public struct AsteriskRule : Rule {
+	public let escapeCharacter : Character? = "\\"
+	public let openTag : String = "*"
+	public let intermediateTag : String? = nil
+	public let closingTag : String? = nil
+	public let maxTags : Int = 3
+	public let minTags : Int = 1
+	public init() { }
+}

Playground/SwiftyMarkdown.playground/Sources/Swifty Tokeniser.swift

@@ -74,6 +74,7 @@ public struct Token {
 	public let inputString : String
 	public var metadataString : String? = nil
 	public var characterStyles : [CharacterStyling] = []
+	public var group : Int = 0
 	public var count : Int = 0
 	public var shouldSkip : Bool = false
 	public var outputString : String {

Playground/SwiftyMarkdown.playground/contents.xcplayground

@@ -5,5 +5,7 @@
         <page name='Line Processing'/>
         <page name='Tokenising'/>
         <page name='Attributed String'/>
+        <page name='SKLabelNode'/>
+        <page name='Groups'/>
     </pages>
 </playground>

Resources/test.md

@@ -62,3 +62,68 @@ Header 2
 	1. Including indented lists
 		- Up to three levels
 1. Neat! 
+
+# SwiftyMarkdown 1.0
+
+SwiftyMarkdown converts Markdown files and strings into `NSAttributedString`s using sensible defaults and a Swift-style syntax. It uses dynamic type to set the font size correctly with whatever font you'd like to use.
+
+## Fully Rebuilt For 2020!
+
+SwiftyMarkdown now features a more robust and reliable rules-based line processing and tokenisation engine. It has added support for images stored in the bundle (`![Image](<Name In bundle>)`), codeblocks, blockquotes, and unordered lists!
+
+Line-level attributes can now have a paragraph alignment applied to them (e.g. `h2.aligment = .center`), and links can be underlined by setting underlineLinks to `true`. 
+
+It also uses the system color `.label` as the default font color on iOS 13 and above for Dark Mode support out of the box. 
+
+## Installation
+
+### CocoaPods:
+
+`pod 'SwiftyMarkdown'`
+
+### SPM: 
+
+In Xcode, `File -> Swift Packages -> Add Package Dependency` and add the GitHub URL. 
+
+*italics* or _italics_
+**bold** or __bold__
+~~Linethrough~~Strikethroughs. 
+`code`
+
+# Header 1
+
+or
+
+Header 1
+====
+
+## Header 2
+
+or
+
+Header 2
+---
+
+### Header 3
+#### Header 4
+##### Header 5 #####
+###### Header 6 ######
+
+	Indented code blocks (spaces or tabs)
+
+[Links](http://voyagetravelapps.com/)
+![Images](<Name of asset in bundle>)
+
+> Blockquotes
+
+- Bulleted
+- Lists
+	- Including indented lists
+		- Up to three levels
+- Neat!
+
+1. Ordered
+1. Lists
+	1. Including indented lists
+		- Up to three levels
+1. Neat! 

Sources/SwiftyMarkdown/SwiftyTokeniser.swift

@@ -28,9 +28,9 @@ public enum SpaceAllowed {
 }
 
 public enum Cancel {
-    case none
+	case none
-    case allRemaining
+	case allRemaining
-    case currentSet
+	case currentSet
 }
 
 public struct CharacterRule : CustomStringConvertible {
@@ -44,6 +44,8 @@ public struct CharacterRule : CustomStringConvertible {
 	public var spacesAllowed : SpaceAllowed = .oneSide
 	public var cancels : Cancel = .none
 	
+	public var tagVarieties : [Int : String]
+	
 	public var description: String {
 		return "Character Rule with Open tag: \(self.openTag) and current styles : \(self.styles) "
 	}
@@ -57,6 +59,11 @@ public struct CharacterRule : CustomStringConvertible {
 		self.minTags = minTags
 		self.maxTags = maxTags
 		self.cancels = cancels
+		
+		self.tagVarieties = [:]
+		for i in minTags...maxTags {
+			self.tagVarieties[i] = openTag.repeating(i)
+		}
 	}
 }
 
@@ -77,6 +84,7 @@ public struct Token {
 	public let id = UUID().uuidString
 	public let type : TokenType
 	public let inputString : String
+	public fileprivate(set) var group : Int = 0
 	public fileprivate(set) var metadataString : String? = nil
 	public fileprivate(set) var characterStyles : [CharacterStyling] = []
 	public fileprivate(set) var count : Int = 0
@@ -84,6 +92,8 @@ public struct Token {
 	public fileprivate(set) var tokenIndex : Int = -1
 	public fileprivate(set) var isProcessed : Bool = false
 	public fileprivate(set) var isMetadata : Bool = false
+	
+	
 	public var outputString : String {
 		get {
 			switch self.type {
@@ -107,6 +117,9 @@ public struct Token {
 		self.type = type
 		self.inputString = inputString
 		self.characterStyles = characterStyles
+		if type == .repeatingTag {
+			self.count = inputString.count
+		}
 	}
 	
 	func newToken( fromSubstring string: String,  isReplacement : Bool) -> Token {
@@ -119,15 +132,220 @@ public struct Token {
 }
 
 extension Sequence where Iterator.Element == Token {
-    var oslogDisplay: String {
+	var oslogDisplay: String {
 		return "[\"\(self.map( {  ($0.outputString.isEmpty) ? "\($0.type): \($0.inputString)" : $0.outputString }).joined(separator: "\", \""))\"]"
-    }
+	}
+}
+
+enum TagState {
+	case none
+	case open
+	case intermediate
+	case closed
+}
+
+struct TagString {
+	var state : TagState = .none
+	var preOpenString = ""
+	var openTagString : [String] = []
+	var intermediateString = ""
+	var intermediateTagString = ""
+	var metadataString = ""
+	var closedTagString : [String] = []
+	var postClosedString = ""
+	
+	let rule : CharacterRule
+	var tokenGroup = 0
+	
+	init( with rule : CharacterRule ) {
+		self.rule = rule
+	}
+	
+	mutating func append( _ string : String? ) {
+		guard let existentString = string else {
+			return
+		}
+		switch self.state {
+		case .none:
+			self.preOpenString += existentString
+		case .open:
+			self.intermediateString += existentString
+		case .intermediate:
+			self.metadataString += existentString
+		case .closed:
+			self.postClosedString += existentString
+		}
+	}
+	
+	mutating func append( contentsOf tokenGroup: [TokenGroup] ) {
+		print(tokenGroup)
+		var availableCount = 0
+		for token in tokenGroup {
+			
+			switch token.state {
+			case .none:
+				self.append(token.string)
+				if self.state == .closed {
+					self.state = .none
+				}
+			case .open:
+				switch self.state {
+				case .none:
+					self.openTagString.append(token.string)
+					self.state = .open
+					availableCount = self.rule.maxTags - 1
+				case .open:
+					if self.rule.closingTag == nil {
+						if availableCount > 0 {
+							self.openTagString.append(token.string)
+							availableCount -= 1
+						} else {
+							self.closedTagString.append(token.string)
+							self.state = .closed
+						}
+					} else if self.rule.maxTags == 1, self.openTagString.first == rule.openTag {
+						self.preOpenString = self.preOpenString + self.openTagString.joined() + self.intermediateString
+						self.intermediateString = ""
+						self.openTagString.append(token.string)
+					} else {
+						self.openTagString.append(token.string)
+					}
+				case .intermediate:
+					self.preOpenString += self.openTagString.joined() + token.string
+				case .closed:
+					self.openTagString.append(token.string)
+				}
+			case .intermediate:
+				switch self.state {
+				case .none:
+					self.preOpenString += token.string
+				case .open:
+					self.intermediateTagString += token.string
+					self.state = .intermediate
+				case .intermediate:
+					self.metadataString += token.string
+				case .closed:
+					self.postClosedString += token.string
+				}
+				
+			case .closed:
+				switch self.state {
+				case .intermediate:
+					self.closedTagString.append(token.string)
+					self.state = .closed
+				case .closed:
+					self.postClosedString += token.string
+				case .open:
+					if self.rule.intermediateTag == nil {
+						self.closedTagString.append(token.string)
+						self.state = .closed
+					} else {
+						self.preOpenString += self.openTagString.joined()
+						self.preOpenString += self.intermediateString
+						self.preOpenString += token.string
+						self.intermediateString = ""
+						self.openTagString.removeAll()
+					}
+				case .none:
+					self.preOpenString += token.string
+				}
+			}
+		}
+		if !self.openTagString.isEmpty && self.rule.closingTag == nil && self.state != .closed {
+			self.state = .open
+		}
+	}
+	
+	func configureToken(ofType type : TokenType = .string, with string : String ) -> Token {
+		var token = Token(type: type, inputString: string)
+		token.group = self.tokenGroup
+		return token
+	}
+	
+	mutating func reset() {
+		self.preOpenString = ""
+		self.openTagString.removeAll()
+		self.intermediateString = ""
+		self.intermediateTagString = ""
+		self.metadataString = ""
+		self.closedTagString.removeAll()
+		self.postClosedString = ""
+		
+		self.state = .none
+	}
+	
+	mutating func tokens(beginningGroupNumberAt group : Int = 0) -> [Token] {
+		print(self)
+		self.tokenGroup = group
+		var tokens : [Token] = []
+		
+		if self.intermediateString.isEmpty && self.intermediateTagString.isEmpty && self.metadataString.isEmpty {
+			tokens.append(self.configureToken(with: self.preOpenString + self.openTagString.joined() + self.closedTagString.joined() + self.postClosedString))
+			self.reset()
+			return tokens
+		}
+		
+		if !self.preOpenString.isEmpty {
+			tokens.append(self.configureToken(with: self.preOpenString))
+		}
+		
+		for tag in self.openTagString {
+			if self.rule.closingTag == nil {
+				tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
+			} else {
+				tokens.append(self.configureToken(ofType: .openTag, with: tag))
+			}
+		}
+		self.tokenGroup += 1
+		if !self.intermediateString.isEmpty {
+			var token = self.configureToken(with: self.intermediateString)
+			token.metadataString = self.metadataString
+			tokens.append(token)
+		}
+		if !self.intermediateTagString.isEmpty {
+			tokens.append(self.configureToken(ofType: .intermediateTag, with: self.intermediateTagString))
+		}
+		
+		self.tokenGroup += 1
+		
+		if !self.metadataString.isEmpty {
+			tokens.append(self.configureToken(with: self.metadataString))
+		}
+		for tag in self.closedTagString {
+			if self.rule.closingTag == nil {
+				tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
+			} else {
+				tokens.append(self.configureToken(ofType: .closeTag, with: tag))
+			}
+		}
+		if !self.postClosedString.isEmpty {
+			tokens.append(self.configureToken(with: self.postClosedString))
+		}
+		
+		self.reset()
+		
+		return tokens
+	}
+}
+
+struct TokenGroup {
+	enum TokenGroupType {
+		case string
+		case tag
+		case escape
+	}
+	
+	let string : String
+	let isEscaped : Bool
+	let type : TokenGroupType
+	var state : TagState = .none
 }
 
 public class SwiftyTokeniser {
 	let rules : [CharacterRule]
 	var replacements : [String : [Token]] = [:]
 	
+	var timer : TimeInterval = 0
 	var enableLog = (ProcessInfo.processInfo.environment["SwiftyTokeniserLogging"] != nil)
 	
 	public init( with rules : [CharacterRule] ) {
@@ -150,11 +368,12 @@ public class SwiftyTokeniser {
 		guard rules.count > 0 else {
 			return [Token(type: .string, inputString: inputString)]
 		}
-
+		
 		var currentTokens : [Token] = []
 		var mutableRules = self.rules
 		
-		
+		self.timer = Date().timeIntervalSinceReferenceDate
+		//		os_log("TIMER BEGIN: 0", log: .tokenising, type: .info)
 		
 		while !mutableRules.isEmpty {
 			let nextRule = mutableRules.removeFirst()
@@ -163,7 +382,7 @@ public class SwiftyTokeniser {
 				os_log("------------------------------", log: .tokenising, type: .info)
 				os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description)
 			}
-	
+			
 			if currentTokens.isEmpty {
 				// This means it's the first time through
 				currentTokens = self.applyStyles(to: self.scan(inputString, with: nextRule), usingRule: nextRule)
@@ -224,7 +443,7 @@ public class SwiftyTokeniser {
 			// Each string could have additional tokens within it, so they have to be scanned as well with the current rule.
 			// The one string token might then be exploded into multiple more tokens
 		}
-
+		
 		if enableLog {
 			os_log("=====RULE PROCESSING COMPLETE=====", log: .tokenising, type: .info)
 			os_log("==================================", log: .tokenising, type: .info)
@@ -389,7 +608,7 @@ public class SwiftyTokeniser {
 	///   - incomingTokens: A group of tokens whose string tokens and replacement tokens should be combined and re-tokenised
 	///   - rule: The current rule being processed
 	func handleReplacementTokens( _ incomingTokens : [Token], with rule : CharacterRule) -> [Token] {
-	
+		
 		// Only combine string and replacements that are next to each other.
 		var newTokenSet : [Token] = []
 		var currentTokenSet : [Token] = []
@@ -409,7 +628,7 @@ public class SwiftyTokeniser {
 			currentTokenSet.append(incomingTokens[i])
 		}
 		newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
-
+		
 		return newTokenSet
 	}
 	
@@ -439,7 +658,7 @@ public class SwiftyTokeniser {
 				}
 			}
 		}
-
+		
 		var metadataString : String = ""
 		for i in metadataIndex..<closeTokenIdx {
 			if tokens[i].type == .string {
@@ -475,7 +694,7 @@ public class SwiftyTokeniser {
 		let theToken = tokens[index]
 		
 		if enableLog {
-		os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
+			os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
 		}
 		
 		guard theToken.count > 0 else {
@@ -487,7 +706,7 @@ public class SwiftyTokeniser {
 		
 		let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
 		// Try to find exact match first
-		if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed }) {
+		if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) {
 			endIdx = nextTokenIdx
 		}
 		
@@ -534,9 +753,9 @@ public class SwiftyTokeniser {
 			let token = mutableTokens[idx]
 			switch token.type {
 			case .escape:
-			if enableLog {
+				if enableLog {
-				os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
+					os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
-			}
+				}
 			case .repeatingTag:
 				let theToken = mutableTokens[idx]
 				self.handleClosingTagFromRepeatingTag(withIndex: idx, in: &mutableTokens, following: rule)
@@ -548,7 +767,7 @@ public class SwiftyTokeniser {
 				if enableLog {
 					os_log("Found open tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
 				}
-								
+				
 				guard rule.closingTag != nil else {
 					
 					// If there's an intermediate tag, get the index of that
@@ -568,8 +787,8 @@ public class SwiftyTokeniser {
 				
 			case .closeTag:
 				let theToken = mutableTokens[idx]
-					if enableLog {
+				if enableLog {
-						os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
+					os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
 				}
 				
 			case .string:
@@ -594,13 +813,89 @@ public class SwiftyTokeniser {
 		return mutableTokens
 	}
 	
-	enum TagState {
+	
-		case open
+	func scanSpacing( _ scanner : Scanner, usingCharactersIn set : CharacterSet ) -> (preTag : String?, foundChars : String?, postTag : String?) {
-		case intermediate
+		let lastChar : String?
-		case closed
+		if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+			lastChar = ( scanner.currentIndex > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
+		} else {
+			if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
+				lastChar = ( scanLocation > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanLocation)..<scanLocation]) : nil
+			} else {
+				lastChar = nil
+			}
+			
+		}
+		let maybeFoundChars : String?
+		if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
+			maybeFoundChars = scanner.scanCharacters(from: set )
+		} else {
+			var string : NSString?
+			scanner.scanCharacters(from: set, into: &string)
+			maybeFoundChars = string as String?
+		}
+		
+		let nextChar : String?
+		if #available(iOS 13.0, OSX 10.15,  watchOS 6.0,tvOS 13.0, *) {
+			nextChar = (scanner.currentIndex != scanner.string.endIndex) ? String(scanner.string[scanner.currentIndex]) : nil
+		} else {
+			if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
+				nextChar = (scanLocation != scanner.string.endIndex) ? String(scanner.string[scanLocation]) : nil
+			} else {
+				nextChar = nil
+			}
+		}
+		return (lastChar, maybeFoundChars, nextChar)
+	}
+	
+	func getTokenGroups( for string : inout String, with rule : CharacterRule, shouldEmpty : Bool = false ) -> [TokenGroup] {
+		if string.isEmpty {
+			return []
+		}
+		var groups : [TokenGroup] 	= []
+		
+		if string.contains(rule.openTag) {
+			if shouldEmpty || string == rule.tagVarieties[rule.maxTags]{
+				var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+				token.state = .open
+				groups.append(token)
+				string.removeAll()
+			}
+			
+		} else if let intermediateString = rule.intermediateTag, string.contains(intermediateString)  {
+			
+			if let range = string.range(of: intermediateString) {
+				let prior = string[string.startIndex..<range.lowerBound]
+				let tag = string[range]
+				let following = string[range.upperBound..<string.endIndex]
+				if !prior.isEmpty {
+					groups.append(TokenGroup(string: String(prior), isEscaped: false, type: .string))
+				}
+				var token = TokenGroup(string: String(tag), isEscaped: false, type: .tag)
+				token.state = .intermediate
+				groups.append(token)
+				if !following.isEmpty {
+					groups.append(TokenGroup(string: String(following), isEscaped: false, type: .string))
+				}
+				string.removeAll()
+			}
+		} else if let closingTag = rule.closingTag, closingTag.contains(string) {
+			var token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			token.state = .closed
+			groups.append(token)
+			string.removeAll()
+		}
+		
+		if shouldEmpty && !string.isEmpty {
+			let token = TokenGroup(string: string, isEscaped: false, type: .tag)
+			groups.append(token)
+			string.removeAll()
+		}
+		return groups
 	}
 	
 	func scan( _ string : String, with rule : CharacterRule) -> [Token] {
+		os_log("TIMER CHECK: %f for rule with openTag %@", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg, rule.openTag)
 		let scanner = Scanner(string: string)
 		scanner.charactersToBeSkipped = nil
 		var tokens : [Token] = []
@@ -610,192 +905,104 @@ public class SwiftyTokeniser {
 		}
 		
 		
-		var openTagFound : TagState = .open
+		var tagString = TagString(with: rule)
-		var openingString = ""
+		var tokenGroup = 0
 		while !scanner.isAtEnd {
-			
+			tokenGroup += 1
 			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
 				if let start = scanner.scanUpToCharacters(from: set) {
-					openingString.append(start)
+					tagString.append(start)
 				}
 			} else {
 				var string : NSString?
 				scanner.scanUpToCharacters(from: set, into: &string)
 				if let existentString = string as String? {
-					openingString.append(existentString)
+					tagString.append(existentString)
 				}
-				// Fallback on earlier versions
 			}
 			
-			let lastChar : String?
+			//				os_log("TIMER CHECK (pre-spacing): %f", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg)
-			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				lastChar = ( scanner.currentIndex > string.startIndex ) ? String(string[string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
-			} else {
-				if let scanLocation = string.index(string.startIndex, offsetBy: scanner.scanLocation, limitedBy: string.endIndex) {
-					lastChar = ( scanLocation > string.startIndex ) ? String(string[string.index(before: scanLocation)..<scanLocation]) : nil
-				} else {
-					lastChar = nil
-				}
-				
-			}
-			let maybeFoundChars : String?
-			if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
-				maybeFoundChars = scanner.scanCharacters(from: set )
-			} else {
-				var string : NSString?
-				scanner.scanCharacters(from: set, into: &string)
-				maybeFoundChars = string as String?
-			}
 			
-			let nextChar : String?
+			// The end of the string
-			if #available(iOS 13.0, OSX 10.15,  watchOS 6.0,tvOS 13.0, *) {
+			let spacing = self.scanSpacing(scanner, usingCharactersIn: set)
-				 nextChar = (scanner.currentIndex != string.endIndex) ? String(string[scanner.currentIndex]) : nil
-			} else {
-				if let scanLocation = string.index(string.startIndex, offsetBy: scanner.scanLocation, limitedBy: string.endIndex) {
-					nextChar = (scanLocation != string.endIndex) ? String(string[scanLocation]) : nil
-				} else {
-					nextChar = nil
-				}
-				
-			}
 			
-			guard let foundChars = maybeFoundChars else {
+			
-				tokens.append(Token(type: .string, inputString: "\(openingString)"))
+			guard let foundTag = spacing.foundChars else {
-				openingString = ""
 				continue
 			}
 			
-			if foundChars == rule.openTag && foundChars.count < rule.minTags {
+			if foundTag == rule.openTag && foundTag.count < rule.minTags {
-				openingString.append(foundChars)
+				tagString.append(foundTag)
 				continue
 			}
 			
-			if !validateSpacing(nextCharacter: nextChar, previousCharacter: lastChar, with: rule) {
+			if !validateSpacing(nextCharacter: spacing.postTag, previousCharacter: spacing.preTag, with: rule) {
 				let escapeString = String("\(rule.escapeCharacter ?? Character(""))")
-				var escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
+				var escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
 				if let hasIntermediateTag = rule.intermediateTag {
-					escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
+					escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
 				}
 				if let existentClosingTag = rule.closingTag {
-					escaped = foundChars.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
+					escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
 				}
-				
+				tagString.append(escaped)
-				openingString.append(escaped)
 				continue
 			}
-
-			// Here's where we have to do the actual tag management.
 			
-			var cumulativeString = ""
+			//				os_log("TIMER CHECK (pre grouping): %f", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg)
-			var openString = ""
+			//:--
-			var containedText = ""
+			print(foundTag)
-			var intermediateString = ""
-			var metadataText = ""
-			var closedString = ""
-			var maybeEscapeNext = false
 			
-			
+			if !foundTag.contains(rule.openTag) && !foundTag.contains(rule.intermediateTag ?? "") && !foundTag.contains(rule.closingTag ?? "") {
-			func addToken( for type : TokenType ) {
+				tagString.append(foundTag)
-				var inputString : String
+				continue
-				switch type {
-				case .openTag:
-					inputString = openString
-				case .intermediateTag:
-					inputString = intermediateString
-				case .closeTag:
-					inputString = closedString
-				default:
-					inputString = ""
-				}
-				guard !inputString.isEmpty else {
-					return
-				}
-				guard inputString.count >= rule.minTags else {
-					return
-				}
-				
-				if !openingString.isEmpty {
-					tokens.append(Token(type: .string, inputString: "\(openingString)"))
-					openingString = ""
-				}
-				let actualType : TokenType = ( rule.intermediateTag == nil && rule.closingTag == nil ) ? .repeatingTag : type
-				
-				var token = Token(type: actualType, inputString: inputString)
-				if rule.closingTag == nil {
-					token.count = inputString.count
-				}
-				
-				tokens.append(token)
-				
-				switch type {
-				case .openTag:
-					openString = ""
-				case .intermediateTag:
-					intermediateString = ""
-				case .closeTag:
-					closedString = ""
-				default:
-					break
-				}
 			}
 			
-			// Here I am going through and adding the characters in the found set to a cumulative string.
+			
-			// If there is an escape character, then the loop stops and any open tags are tokenised.
+			var tokenGroups : [TokenGroup] = []
-			for char in foundChars {
+			var escapeCharacter : Character? = nil
-				cumulativeString.append(char)
+			var cumulatedString = ""
-				if maybeEscapeNext {
+			for char in foundTag {
+				if let existentEscapeCharacter = escapeCharacter {
 					
-					var escaped = cumulativeString
+					// If any of the tags feature the current character
-					if String(char) == rule.openTag || String(char) == rule.intermediateTag || String(char) == rule.closingTag {
+					let escape = String(existentEscapeCharacter)
-						escaped = String(cumulativeString.replacingOccurrences(of: String(rule.escapeCharacter ?? Character("")), with: ""))
+					let nextTagCharacter = String(char)
+					if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
+						escapeCharacter = nil
+					} else if nextTagCharacter == escape {
+						// Doesn't apply to this rule
+						tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
 					}
 					
-					openingString.append(escaped)
+					continue
-					cumulativeString = ""
-					maybeEscapeNext = false
 				}
 				if let existentEscape = rule.escapeCharacter {
-					if cumulativeString == String(existentEscape) {
+					if char == existentEscape {
-						maybeEscapeNext = true
+						tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
-						addToken(for: .openTag)
+						escapeCharacter = char
-						addToken(for: .intermediateTag)
-						addToken(for: .closeTag)
 						continue
 					}
 				}
+				cumulatedString.append(char)
+				tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule))
 				
-				
-				if cumulativeString == rule.openTag, openTagFound == .open {
-					openString.append(char)
-					cumulativeString = ""
-					openTagFound = ( rule.closingTag == nil ) ? .open : .closed
-					openTagFound = ( rule.intermediateTag == nil ) ? openTagFound : .intermediate
-				} else if cumulativeString == rule.intermediateTag, openTagFound == .intermediate {
-					intermediateString.append(cumulativeString)
-					cumulativeString = ""
-					openTagFound = ( rule.closingTag == nil ) ? .open : .closed
-				} else if cumulativeString == rule.closingTag, openTagFound == .closed {
-					closedString.append(char)
-					cumulativeString = ""
-					openTagFound = .open
-				}
 			}
-
+			if let remainingEscape = escapeCharacter {
+				tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
+			}
 			
-			addToken(for: .openTag)
+			tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
-			addToken(for: .intermediateTag)
+			tagString.append(contentsOf: tokenGroups)
-			addToken(for: .closeTag)
-			openingString.append( cumulativeString )
 			
-			// If we're here, it means that an escape character was found but without a corresponding
+			if tagString.state == .none {
-			// tag, which means it might belong to a different rule.
+				tokens.append(contentsOf: tagString.tokens(beginningGroupNumberAt : tokenGroup))
-			// It should be added to the next group of regular characters
+			}
-
 		}
 		
-		if !openingString.isEmpty {
+		tokens.append(contentsOf: tagString.tokens(beginningGroupNumberAt : tokenGroup))
-			tokens.append(Token(type: .string, inputString: "\(openingString)"))
+		//			os_log("TIMER TOTAL: %f for rule with openTag %@", log: .tokenising, type: .info, Date().timeIntervalSinceReferenceDate - self.timer as CVarArg, rule.openTag)
-		}
 		
 		return tokens
 	}
@@ -823,7 +1030,7 @@ public class SwiftyTokeniser {
 			default:
 				return true
 			}
-		
+			
 		case .oneSide:
 			switch (previousCharacter, nextCharacter) {
 			case  (nil, " " ), (" ", nil), (" ", " " ):
@@ -838,3 +1045,14 @@ public class SwiftyTokeniser {
 	}
 	
 }
+
+
+extension String {
+	func repeating( _ max : Int ) -> String {
+		var output = self
+		for _ in 1..<max {
+			output += self
+		}
+		return output
+	}
+}

SwiftyMarkdown.xcodeproj/xcshareddata/xcbaselines/SwiftyMarkdown::SwiftyMarkdownTests.xcbaseline/AD1DF83E-20BC-4E7E-8C14-683818ED0A26.plist

@@ -11,9 +11,11 @@
 				<key>com.apple.XCTPerformanceMetric_WallClockTime</key>
 				<dict>
 					<key>baselineAverage</key>
-					<real>0.0217</real>
+					<real>0.01</real>
 					<key>baselineIntegrationDisplayName</key>
 					<string>Local Baseline</string>
+					<key>maxPercentRelativeStandardDeviation</key>
+					<real>10</real>
 				</dict>
 			</dict>
 			<key>testThatStringsAreProcessedQuickly()</key>

Tests/SwiftyMarkdownTests/SwiftyMarkdownCharacterTests.swift

@@ -12,36 +12,50 @@ import XCTest
 class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	
 	func testIsolatedCase() {
-		let challenge = TokenTest(input: "An *\\*italic\\** [referenced link][link]", output: "An *italic* referenced link", tokens: [
+
+		challenge = TokenTest(input: "A string with a **bold** word", output: "A string with a bold word",  tokens: [
+			Token(type: .string, inputString: "A string with a ", characterStyles: []),
+			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
+			Token(type: .string, inputString: " word", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		return
+		
+		challenge = TokenTest(input: "An *\\*italic\\** [referenced link][link]", output: "An *italic* referenced link", tokens: [
 			Token(type: .string, inputString: "An ", characterStyles: []),
 			Token(type: .string, inputString: "*italic*", characterStyles: [CharacterStyle.italic]),
 			Token(type: .string, inputString: " ", characterStyles: []),
 			Token(type: .string, inputString: "referenced link", characterStyles: [CharacterStyle.link])
 		])
-		let rules : [CharacterRule] = [
+		rules  = [
 			CharacterRule(openTag: "*", escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [.bold], 3 : [.italic, .bold]], minTags: 1, maxTags: 3),
 			CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], minTags: 1, maxTags: 1),
 			CharacterRule(openTag: "[", intermediateTag: "][", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], minTags: 1, maxTags: 1)
 		]
-		let results = self.attempt(challenge, rules: rules)
+		results = self.attempt(challenge, rules: rules)
 		XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
-		var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
 		
-		if links.count == 1 {
+		
-			XCTAssertEqual(links[0].metadataString, "https://www.neverendingvoyage.com/")
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataString, "https://www.neverendingvoyage.com/")
 		} else {
-			XCTFail("Incorrect link count. Expecting 1, found \(links.count)")
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
 		}
 		
 	}
 	
 	func testThatBoldTraitsAreRecognised() {
-		var challenge = TokenTest(input: "**A bold string**", output: "A bold string",  tokens: [
+		challenge = TokenTest(input: "**A bold string**", output: "A bold string",  tokens: [
 			Token(type: .string, inputString: "A bold string", characterStyles: [CharacterStyle.bold])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -67,6 +81,15 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
+		challenge = TokenTest(input: "\\\\*\\*A normal \\\\ string\\*\\*", output: "\\**A normal \\\\ string**", tokens: [
+			Token(type: .string, inputString: "\\**A normal \\\\ string**", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
 		challenge = TokenTest(input: "A string with double \\*\\*escaped\\*\\* asterisks", output: "A string with double **escaped** asterisks", tokens: [
 			Token(type: .string, inputString: "A string with double **escaped** asterisks", characterStyles: [])
 		])
@@ -99,10 +122,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	}
 	
 	func testThatCodeTraitsAreRecognised() {
-		var challenge = TokenTest(input: "`Code (**should** not process internal tags)`", output: "Code (**should** not process internal tags)",  tokens: [
+		challenge = TokenTest(input: "`Code (**should** not process internal tags)`", output: "Code (**should** not process internal tags)",  tokens: [
 			Token(type: .string, inputString: "Code (**should** not process internal tags) ", characterStyles: [CharacterStyle.code])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -158,14 +181,23 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
+		challenge = TokenTest(input: "Two backticks followed by a full stop ``.", output: "Two backticks followed by a full stop ``.", tokens: [
+			Token(type: .string, inputString: "Two backticks followed by a full stop ``.", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
 	}
 	
 	func testThatItalicTraitsAreParsedCorrectly() {
 		
-		var challenge = TokenTest(input: "*An italicised string*", output: "An italicised string", tokens : [
+		challenge = TokenTest(input: "*An italicised string*", output: "An italicised string", tokens : [
 			Token(type: .string, inputString: "An italicised string", characterStyles: [CharacterStyle.italic])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -236,11 +268,11 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	}
 	
 	func testThatStrikethroughTraitsAreRecognised() {
-		var challenge = TokenTest(input: "~~An~~A crossed-out string", output: "AnA crossed-out string", tokens: [
+		challenge = TokenTest(input: "~~An~~A crossed-out string", output: "AnA crossed-out string", tokens: [
 			Token(type: .string, inputString: "An", characterStyles: [CharacterStyle.strikethrough]),
 			Token(type: .string, inputString: "A crossed-out string", characterStyles: [])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -269,7 +301,7 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	
 	func testThatMixedTraitsAreRecognised() {
 		
-		var challenge = TokenTest(input: "__A bold string__ with a **mix** **of** bold __styles__", output: "A bold string with a mix of bold styles", tokens : [
+		challenge = TokenTest(input: "__A bold string__ with a **mix** **of** bold __styles__", output: "A bold string with a mix of bold styles", tokens : [
 			Token(type: .string, inputString: "A bold string", characterStyles: [CharacterStyle.bold]),
 			Token(type: .string, inputString: "with a ", characterStyles: []),
 			Token(type: .string, inputString: "mix", characterStyles: [CharacterStyle.bold]),
@@ -278,7 +310,7 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 			Token(type: .string, inputString: " bold ", characterStyles: []),
 			Token(type: .string, inputString: "styles", characterStyles: [CharacterStyle.bold])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -306,10 +338,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	}
 	
 	func testThatExtraCharactersAreHandles() {
-		var challenge = TokenTest(input: "***A bold italic string***", output: "A bold italic string",  tokens: [
+		challenge = TokenTest(input: "***A bold italic string***", output: "A bold italic string",  tokens: [
 			Token(type: .string, inputString: "A bold italic string", characterStyles: [CharacterStyle.bold, CharacterStyle.italic])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -380,12 +412,12 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	
 	func offtestAdvancedEscaping() {
 		
-		var challenge = TokenTest(input: "\\***A normal string*\\**", output: "**A normal string*", tokens: [
+		challenge = TokenTest(input: "\\***A normal string*\\**", output: "**A normal string*", tokens: [
 			Token(type: .string, inputString: "**", characterStyles: []),
 			Token(type: .string, inputString: "A normal string", characterStyles: [CharacterStyle.italic]),
 			Token(type: .string, inputString: "**", characterStyles: [])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -411,7 +443,6 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 		let asteriskComma = "An asterisk followed by a full stop: *, *"
 		
 		let backtickSpace = "A backtick followed by a space: `"
-		let backtickFullStop = "Two backticks followed by a full stop: ``."
 		
 		let underscoreSpace = "An underscore followed by a space: _"
 
@@ -431,9 +462,6 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 		md = SwiftyMarkdown(string: asteriskFullStop)
 		XCTAssertEqual(md.attributedString().string, asteriskFullStop)
 		
-		md = SwiftyMarkdown(string: backtickFullStop)
-		XCTAssertEqual(md.attributedString().string, backtickFullStop)
-		
 		md = SwiftyMarkdown(string: underscoreFullStop)
 		XCTAssertEqual(md.attributedString().string, underscoreFullStop)
 		
@@ -458,10 +486,10 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
 	}
 	
 	func testReportedCrashingStrings() {
-		let challenge = TokenTest(input: "[**\\!bang**](https://duckduckgo.com/bang)", output: "\\!bang", tokens: [
+		challenge = TokenTest(input: "[**\\!bang**](https://duckduckgo.com/bang)", output: "\\!bang", tokens: [
 			Token(type: .string, inputString: "\\!bang", characterStyles: [CharacterStyle.bold, CharacterStyle.link])
 		])
-		let results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)

Tests/SwiftyMarkdownTests/SwiftyMarkdownLinkTests.swift

@@ -13,10 +13,10 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 	
 	func testForLinks() {
 		
-		var challenge = TokenTest(input: "[Link at start](http://voyagetravelapps.com/)", output: "Link at start", tokens: [
+		challenge = TokenTest(input: "[Link at start](http://voyagetravelapps.com/)", output: "Link at start", tokens: [
 			Token(type: .string, inputString: "Link at start", characterStyles: [CharacterStyle.link])
 		])
-		var results = self.attempt(challenge)
+		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
@@ -82,25 +82,10 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 		}
 
 	
-		challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/)", tokens: [
-			Token(type: .string, inputString: "Link with missing square(http://voyagetravelapps.com/)", characterStyles: [])
-		])
-		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
-		XCTAssertEqual(results.foundStyles, results.expectedStyles)
-		XCTAssertEqual(results.attributedString.string, challenge.output)
-		
-		challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/)", tokens: [
-			Token(type: .string, inputString: "A ", characterStyles: []),
-			Token(type: .string, inputString: "[Link(http://voyagetravelapps.com/)", characterStyles: [])
-		])
-		results = self.attempt(challenge)
-		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
-		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
-		XCTAssertEqual(results.foundStyles, results.expectedStyles)
-		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
+	}
+	
+	func testMalformedLinks() {
 		
 		challenge = TokenTest(input: "[Link with missing parenthesis](http://voyagetravelapps.com/", output: "[Link with missing parenthesis](http://voyagetravelapps.com/", tokens: [
 			Token(type: .string, inputString: "[Link with missing parenthesis](", characterStyles: []),
@@ -123,19 +108,61 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
 		
-		challenge = TokenTest(input: "[Link1](http://voyagetravelapps.com/) **bold** [Link2](http://voyagetravelapps.com/)", output: "Link1 bold Link2",  tokens: [
+		challenge = TokenTest(input: "[A link](((url)", output: "A link", tokens: [
-			Token(type: .string, inputString: "Link1", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: "A link", characterStyles: [CharacterStyle.link])
-			Token(type: .string, inputString: " ", characterStyles: []),
+		])
-			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
+		rules = [
-			Token(type: .string, inputString: " ", characterStyles: []),
+			CharacterRule(openTag: "![", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.image]]),
-			Token(type: .string, inputString: "Link2", characterStyles: [CharacterStyle.link])
+			CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]]),
+		]
+		results = self.attempt(challenge, rules: rules)
+		
+		
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataString, "((url")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/)", tokens: [
+			Token(type: .string, inputString: "Link with missing square(http://voyagetravelapps.com/)", characterStyles: [])
 		])
 		results = self.attempt(challenge)
 		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
 		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
 		XCTAssertEqual(results.foundStyles, results.expectedStyles)
 		XCTAssertEqual(results.attributedString.string, challenge.output)
-	
+		
+		challenge = TokenTest(input: "[Link with [second opening](http://voyagetravelapps.com/)", output: "[Link with second opening", tokens: [
+			Token(type: .string, inputString: "Link with ", characterStyles: []),
+			Token(type: .string, inputString: "second opening", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		XCTAssertEqual(results.links.count, 1)
+		if results.links.count == 1 {
+			XCTAssertEqual(results.links[0].metadataString, "http://voyagetravelapps.com/")
+		} else {
+			XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
+		}
+		
+		challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/)", tokens: [
+			Token(type: .string, inputString: "A [Link(http://voyagetravelapps.com/)", characterStyles: [])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
+		
 	}
 	
 	func testLinksWithOtherStyles() {
@@ -173,6 +200,19 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
 		} else {
 			XCTFail("Incorrect link count. Expecting 1, found \(links.count)")
 		}
+		
+		challenge = TokenTest(input: "[Link1](http://voyagetravelapps.com/) **bold** [Link2](http://voyagetravelapps.com/)", output: "Link1 bold Link2",  tokens: [
+			Token(type: .string, inputString: "Link1", characterStyles: [CharacterStyle.link]),
+			Token(type: .string, inputString: " ", characterStyles: []),
+			Token(type: .string, inputString: "bold", characterStyles: [CharacterStyle.bold]),
+			Token(type: .string, inputString: " ", characterStyles: []),
+			Token(type: .string, inputString: "Link2", characterStyles: [CharacterStyle.link])
+		])
+		results = self.attempt(challenge)
+		XCTAssertEqual(challenge.tokens.count, results.stringTokens.count)
+		XCTAssertEqual(results.tokens.map({ $0.outputString }).joined(), challenge.output)
+		XCTAssertEqual(results.foundStyles, results.expectedStyles)
+		XCTAssertEqual(results.attributedString.string, challenge.output)
 	}
 	
 	func testForImages() {

Tests/SwiftyMarkdownTests/XCTest+SwiftyMarkdown.swift

@@ -9,11 +9,41 @@
 import XCTest
 @testable import SwiftyMarkdown
 
+
+struct ChallengeReturn {
+	let tokens : [Token]
+	let stringTokens : [Token]
+	let links : [Token]
+	let attributedString : NSAttributedString
+	let foundStyles : [[CharacterStyle]]
+	let expectedStyles : [[CharacterStyle]]
+}
+
 class SwiftyMarkdownCharacterTests : XCTestCase {
 	let defaultRules = SwiftyMarkdown.characterRules
 	
-	func testDummy() {
+	var challenge : TokenTest!
+	var results : ChallengeReturn!
+	var rules : [CharacterRule]? = nil
+	
+	func attempt( _ challenge : TokenTest, rules : [CharacterRule]? = nil ) -> ChallengeReturn {
+		if let validRules = rules {
+			SwiftyMarkdown.characterRules = validRules
+		} else {
+			SwiftyMarkdown.characterRules = self.defaultRules
+		}
 		
+		let md = SwiftyMarkdown(string: challenge.input)
+		let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
+		let tokens = tokeniser.process(challenge.input)
+		let stringTokens = tokens.filter({ $0.type == .string && !$0.isMetadata })
+		
+		let existentTokenStyles = stringTokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
+		let expectedStyles = challenge.tokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
+		
+		let linkTokens = tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) })
+		
+		return ChallengeReturn(tokens: tokens, stringTokens: stringTokens, links : linkTokens, attributedString:  md.attributedString(), foundStyles: existentTokenStyles, expectedStyles : expectedStyles)
 	}
 }
 
@@ -29,22 +59,4 @@ extension XCTestCase {
 
 }
 
-extension SwiftyMarkdownCharacterTests {
+
-	func attempt( _ challenge : TokenTest, rules : [CharacterRule]? = nil ) -> (tokens : [Token], stringTokens: [Token], attributedString : NSAttributedString, foundStyles : [[CharacterStyle]], expectedStyles : [[CharacterStyle]] ) {
-		if let validRules = rules {
-			SwiftyMarkdown.characterRules = validRules
-		} else {
-			SwiftyMarkdown.characterRules = self.defaultRules
-		}
-		
-		let md = SwiftyMarkdown(string: challenge.input)
-		let tokeniser = SwiftyTokeniser(with: SwiftyMarkdown.characterRules)
-		let tokens = tokeniser.process(challenge.input)
-		let stringTokens = tokens.filter({ $0.type == .string && !$0.isMetadata })
-		
-		let existentTokenStyles = stringTokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
-		let expectedStyles = challenge.tokens.compactMap({ $0.characterStyles as? [CharacterStyle] })
-		
-		return (tokens, stringTokens, md.attributedString(), existentTokenStyles, expectedStyles)
-	}
-}