dnrops
/
SwiftyMarkdown
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Marker before third attempt at parser

This commit is contained in:
Simon Fairbairn 2020-04-08 14:10:26 +12:00
parent 68336ab630
commit ba5d8ce9c6
9 changed files with 709 additions and 1204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Example/SwiftyMarkdownExample/example.md
View File

@ -1 +1 @@
[Link 1](http://voyagetravelapps.com/)[Link 2](https://www.neverendingvoyage.com/)
An ![Image](imageName)

86
Sources/SwiftyMarkdown/CharacterRule.swift
View File

@ -32,18 +32,54 @@ public struct v2_CharacterRule {
public let metadataClose : Character?
}
public struct CharacterRule : CustomStringConvertible {
public let openTag : String
public var closeTag : String? {
get {
return self.closingTag
}
public enum EscapeCharacterRule {
case keep
case remove
}
public struct EscapeCharacter {
let character : Character
let rule : EscapeCharacterRule
public init( character : Character, rule : EscapeCharacterRule ) {
self.character = character
self.rule = rule
}
public let intermediateTag : String?
public let closingTag : String?
public let escapeCharacter : Character?
public let metadataOpen : Character?
public let metadataClose : Character?
}
public enum CharacterRuleTagType {
case open
case close
case metadataOpen
case metadataClose
case repeating
}
public struct CharacterRuleTag {
let tag : String
let escapeCharacters : [EscapeCharacter]
let type : CharacterRuleTagType
let min : Int
let max : Int
public init( tag : String, type : CharacterRuleTagType, escapeCharacters : [EscapeCharacter] = [EscapeCharacter(character: "\\", rule: .remove)], min : Int = 1, max : Int = 1) {
self.tag = tag
self.type = type
self.escapeCharacters = escapeCharacters
self.min = min
self.max = max
}
public func escapeCharacter( for character : Character ) -> EscapeCharacter? {
return self.escapeCharacters.filter({ $0.character == character }).first
}
}
public struct CharacterRule : CustomStringConvertible {
public let tags : [CharacterRuleTag]
public let styles : [Int : [CharacterStyling]]
public var minTags : Int = 1
public var maxTags : Int = 1
@ -51,32 +87,28 @@ public struct CharacterRule : CustomStringConvertible {
public var cancels : Cancel = .none
public var metadataLookup : Bool = false
public var isRepeatingTag : Bool {
return self.closingTag == nil && self.intermediateTag == nil
return self.primaryTag.type == .repeating
}
public var tagVarieties : [Int : String]
public var isSelfContained = false
public var description: String {
return "Character Rule with Open tag: \(self.openTag) and current styles : \(self.styles) "
return "Character Rule with Open tag: \(self.primaryTag.tag) and current styles : \(self.styles) "
}
public init(openTag: String, intermediateTag: String? = nil, closingTag: String? = nil, escapeCharacter: Character? = nil, styles: [Int : [CharacterStyling]] = [:], minTags : Int = 1, maxTags : Int = 1, cancels : Cancel = .none, metadataLookup : Bool = false, spacesAllowed: SpaceAllowed = .oneSide, metadataOpen : Character? = nil, metadataClose : Character? = nil) {
self.openTag = openTag
self.intermediateTag = intermediateTag
self.closingTag = closingTag
self.escapeCharacter = escapeCharacter
public let primaryTag : CharacterRuleTag
public func tag( for type : CharacterRuleTagType ) -> CharacterRuleTag? {
return self.tags.filter({ $0.type == type }).first ?? nil
}
public init(primaryTag: CharacterRuleTag, otherTags: [CharacterRuleTag], styles: [Int : [CharacterStyling]] = [:], cancels : Cancel = .none, metadataLookup : Bool = false, spacesAllowed: SpaceAllowed = .oneSide, isSelfContained : Bool = false) {
self.primaryTag = primaryTag
self.tags = otherTags
self.styles = styles
self.minTags = minTags
self.maxTags = maxTags
self.cancels = cancels
self.metadataLookup = metadataLookup
self.spacesAllowed = spacesAllowed
self.tagVarieties = [:]
self.metadataOpen = metadataOpen
self.metadataClose = metadataClose
for i in minTags...maxTags {
self.tagVarieties[i] = openTag.repeating(i)
}
self.isSelfContained = isSelfContained
}
}

31
Sources/SwiftyMarkdown/SwiftyMarkdown.swift
View File

@ -168,13 +168,25 @@ If that is not set, then the system default will be used.
]
static public var characterRules = [
CharacterRule(openTag: "[", intermediateTag: "][", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], metadataLookup: true, spacesAllowed: .bothSides, metadataOpen: "[", metadataClose: "]"),
CharacterRule(openTag: "![", intermediateTag: "](", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.image]], metadataLookup: false, spacesAllowed: .bothSides, metadataOpen: "(", metadataClose: ")"),
CharacterRule(openTag: "[", intermediateTag: "](", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], maxTags: 1, spacesAllowed: .bothSides, metadataOpen: "(", metadataClose: ")"),
CharacterRule(openTag: "`", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.code]], maxTags: 1, cancels: .allRemaining),
CharacterRule(openTag: "~", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [2 : [CharacterStyle.strikethrough]], minTags: 2, maxTags: 2),
CharacterRule(openTag: "*", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3),
CharacterRule(openTag: "_", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3)
CharacterRule(primaryTag: CharacterRuleTag(tag: "![", type: .open), otherTags: [
CharacterRuleTag(tag: "]", type: .close),
CharacterRuleTag(tag: "(", type: .metadataOpen),
CharacterRuleTag(tag: ")", type: .metadataClose)
], styles: [1 : [CharacterStyle.image]], metadataLookup: false, spacesAllowed: .bothSides, isSelfContained: true),
CharacterRule(primaryTag: CharacterRuleTag(tag: "[", type: .open, escapeCharacters: [EscapeCharacter(character: "\\", rule: .remove),EscapeCharacter(character: "!", rule: .keep)]), otherTags: [
CharacterRuleTag(tag: "]", type: .close),
CharacterRuleTag(tag: "[", type: .metadataOpen),
CharacterRuleTag(tag: "]", type: .metadataClose)
], styles: [1 : [CharacterStyle.referencedLink]], metadataLookup: true, spacesAllowed: .bothSides, isSelfContained: true),
CharacterRule(primaryTag: CharacterRuleTag(tag: "[", type: .open, escapeCharacters: [EscapeCharacter(character: "\\", rule: .remove),EscapeCharacter(character: "!", rule: .keep)]), otherTags: [
CharacterRuleTag(tag: "]", type: .close),
CharacterRuleTag(tag: "(", type: .metadataOpen),
CharacterRuleTag(tag: ")", type: .metadataClose)
], styles: [1 : [CharacterStyle.link]], metadataLookup: true, spacesAllowed: .bothSides, isSelfContained: true),
CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : [CharacterStyle.code]], cancels: .allRemaining),
CharacterRule(primaryTag:CharacterRuleTag(tag: "~", type: .repeating, min: 2, max: 2), otherTags : [], styles: [2 : [CharacterStyle.strikethrough]]),
CharacterRule(primaryTag: CharacterRuleTag(tag: "*", type: .repeating, min: 1, max: 3), otherTags: [], styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]]),
CharacterRule(primaryTag: CharacterRuleTag(tag: "_", type: .repeating, min: 1, max: 3), otherTags: [], styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]])
]
let lineProcessor = SwiftyLineProcessor(rules: SwiftyMarkdown.lineRules, defaultRule: MarkdownLineStyle.body, frontMatterRules: SwiftyMarkdown.frontMatterRules)
@ -238,6 +250,8 @@ If that is not set, then the system default will be used.
var previouslyFoundTokens : [Token] = []
var applyAttachments = true
let perfomanceLog = PerformanceLog(with: "SwiftyMarkdownPerformanceLogging", identifier: "Swifty Markdown", log: .swiftyMarkdownPerformance)
/**
@ -554,6 +568,9 @@ extension SwiftyMarkdown {
#if !os(watchOS)
if styles.contains(.image), let imageName = token.metadataString {
if !self.applyAttachments {
continue
}
#if !os(macOS)
let image1Attachment = NSTextAttachment()
image1Attachment.image = UIImage(named: imageName)

663
Sources/SwiftyMarkdown/SwiftyScanner.swift
View File

@ -18,6 +18,7 @@ extension OSLog {
public protocol SwiftyScanning {
var metadataLookup : [String : String] { get set }
func scan( _ string : String, with rule : CharacterRule) -> [Token]
func scan( _ tokens : [Token], with rule : CharacterRule) -> [Token]
}
enum TagState {
@ -28,664 +29,20 @@ enum TagState {
}
class SwiftyScanner : SwiftyScanning {
var state : TagState = .none
var preOpenString = ""
var openTagString : [String] = []
var intermediateString = ""
var intermediateTagString = ""
var metadataString = ""
var closedTagString : [String] = []
var postClosedString = ""
var metadataLookup: [String : String] = [:]
var rule : CharacterRule! = nil
var tokenGroup = 0
var metadataLookup : [String : String] = [:]
let performanceLog = PerformanceLog(with: "SwiftyScannerPerformanceLogging", identifier: "Swifty Scanner", log: .swiftyScannerPerformance)
init() {
init() { }
func append( _ string : String? ) {
guard let existentString = string else {
return
}
switch self.state {
case .none:
self.preOpenString += existentString
case .open:
self.intermediateString += existentString
case .intermediate:
self.metadataString += existentString
case .closed:
self.postClosedString += existentString
}
}
func handleRepeatingTags( _ tokenGroup : [TokenGroup] ) {
var availableCount = self.rule.maxTags
var sameOpenGroup = false
for token in tokenGroup {
switch token.state {
case .none:
self.append(token.string)
if self.state == .closed {
self.state = .none
}
case .open:
switch self.state {
case .none:
self.openTagString.append(token.string)
self.state = .open
availableCount = self.rule.maxTags - token.string.count
sameOpenGroup = true
case .open:
if availableCount > 0 {
if sameOpenGroup {
self.openTagString.append(token.string)
availableCount = self.rule.maxTags - token.string.count
} else {
self.closedTagString.append(token.string)
self.state = .closed
}
} else {
self.append(token.string)
}
func scan(_ string: String, with rule: CharacterRule) -> [Token] {
return []
}
func scan(_ tokens: [Token], with rule: CharacterRule) -> [Token] {
return tokens
}
case .intermediate:
self.preOpenString += self.openTagString.joined() + token.string
case .closed:
self.append(token.string)
}
case .intermediate:
switch self.state {
case .none:
self.preOpenString += token.string
case .open:
self.intermediateTagString += token.string
self.state = .intermediate
case .intermediate:
self.metadataString += token.string
case .closed:
self.postClosedString += token.string
}
case .closed:
switch self.state {
case .intermediate:
self.closedTagString.append(token.string)
self.state = .closed
case .closed:
self.postClosedString += token.string
case .open:
if self.rule.intermediateTag == nil {
self.closedTagString.append(token.string)
self.state = .closed
} else {
self.preOpenString += self.openTagString.joined()
self.preOpenString += self.intermediateString
self.preOpenString += token.string
self.intermediateString = ""
self.openTagString.removeAll()
}
case .none:
self.preOpenString += token.string
}
}
}
if !self.openTagString.isEmpty && self.rule.closingTag == nil && self.state != .closed {
self.state = .open
}
}
func handleRegularTags( _ tokenGroup : [TokenGroup] ) {
for token in tokenGroup {
switch token.state {
case .none:
self.append(token.string)
if self.state == .closed {
self.state = .none
}
case .open:
switch self.state {
case .none:
self.openTagString.append(token.string)
self.state = .open
case .open:
if self.rule.maxTags == 1, self.openTagString.first == rule.openTag {
self.preOpenString = self.preOpenString + self.openTagString.joined() + self.intermediateString
self.intermediateString = ""
self.openTagString.removeAll()
self.openTagString.append(token.string)
} else {
self.openTagString.append(token.string)
}
case .intermediate:
self.preOpenString += self.openTagString.joined() + token.string
case .closed:
self.openTagString.append(token.string)
}
case .intermediate:
switch self.state {
case .none:
self.preOpenString += token.string
case .open:
self.intermediateTagString += token.string
self.state = .intermediate
case .intermediate:
self.metadataString += token.string
case .closed:
self.postClosedString += token.string
}
case .closed:
switch self.state {
case .intermediate:
self.closedTagString.append(token.string)
self.state = .closed
case .closed:
self.postClosedString += token.string
case .open:
if self.rule.intermediateTag == nil {
self.closedTagString.append(token.string)
self.state = .closed
} else {
self.preOpenString += self.openTagString.joined()
self.preOpenString += self.intermediateString
self.preOpenString += token.string
self.intermediateString = ""
self.openTagString.removeAll()
}
case .none:
self.preOpenString += token.string
}
}
}
}
func append( contentsOf tokenGroup: [TokenGroup] ) {
if self.rule.isRepeatingTag {
self.handleRepeatingTags(tokenGroup)
} else {
self.handleRegularTags(tokenGroup)
}
}
func configureToken(ofType type : TokenType = .string, with string : String ) -> Token {
var token = Token(type: type, inputString: string)
token.group = self.tokenGroup
return token
}
func reset() {
self.preOpenString = ""
self.openTagString.removeAll()
self.intermediateString = ""
self.intermediateTagString = ""
self.metadataString = ""
self.closedTagString.removeAll()
self.postClosedString = ""
self.state = .none
}
func consolidate(with string : String, into tokens : inout [Token]) -> [Token] {
self.reset()
guard !string.isEmpty else {
return tokens
}
tokens.append(self.configureToken(with: string))
return tokens
}
func tokens(beginningGroupNumberAt group : Int = 0) -> [Token] {
self.tokenGroup = group
var tokens : [Token] = []
if self.intermediateString.isEmpty && self.intermediateTagString.isEmpty && self.metadataString.isEmpty {
let actualString = self.preOpenString + self.openTagString.joined() + self.closedTagString.joined() + self.postClosedString
return self.consolidate(with: actualString, into: &tokens)
}
if self.state == .open && !self.openTagString.isEmpty {
let actualString = self.preOpenString + self.openTagString.joined() + self.intermediateString
return self.consolidate(with: actualString, into: &tokens)
}
if !self.preOpenString.isEmpty {
tokens.append(self.configureToken(with: self.preOpenString))
}
for tag in self.openTagString {
if self.rule.isRepeatingTag {
tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
} else {
tokens.append(self.configureToken(ofType: .openTag, with: tag))
}
}
self.tokenGroup += 1
if !self.intermediateString.isEmpty {
var token = self.configureToken(with: self.intermediateString)
token.metadataString = (self.metadataString.isEmpty) ? nil : self.metadataString
tokens.append(token)
}
if !self.intermediateTagString.isEmpty {
tokens.append(self.configureToken(ofType: .intermediateTag, with: self.intermediateTagString))
}
self.tokenGroup += 1
if !self.metadataString.isEmpty {
var token = Token(type: .string, inputString: self.metadataString)
token.group = self.tokenGroup
tokens.append(token)
}
var remainingTags = ( self.rule.closingTag == nil ) ? self.openTagString.joined() : ""
for tag in self.closedTagString {
if self.rule.isRepeatingTag {
remainingTags = remainingTags.replacingOccurrences(of: tag, with: "")
tokens.append(self.configureToken(ofType: .repeatingTag, with: tag))
} else {
tokens.append(self.configureToken(ofType: .closeTag, with: tag))
}
}
if !self.postClosedString.isEmpty {
tokens.append(self.configureToken(with: self.postClosedString))
}
self.reset()
if !remainingTags.isEmpty {
self.state = .open
}
return tokens
}
func scanSpacing( _ scanner : Scanner, usingCharactersIn set : CharacterSet ) -> (preTag : String?, foundChars : String?, postTag : String?) {
self.performanceLog.tag(with: "(scan space)")
let lastChar : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
lastChar = ( scanner.currentIndex > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanner.currentIndex)..<scanner.currentIndex]) : nil
} else {
if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
lastChar = ( scanLocation > scanner.string.startIndex ) ? String(scanner.string[scanner.string.index(before: scanLocation)..<scanLocation]) : nil
} else {
lastChar = nil
}
}
let maybeFoundChars : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
maybeFoundChars = scanner.scanCharacters(from: set )
} else {
var string : NSString?
scanner.scanCharacters(from: set, into: &string)
maybeFoundChars = string as String?
}
let nextChar : String?
if #available(iOS 13.0, OSX 10.15, watchOS 6.0,tvOS 13.0, *) {
nextChar = (scanner.currentIndex != scanner.string.endIndex) ? String(scanner.string[scanner.currentIndex]) : nil
} else {
if let scanLocation = scanner.string.index(scanner.string.startIndex, offsetBy: scanner.scanLocation, limitedBy: scanner.string.endIndex) {
nextChar = (scanLocation != scanner.string.endIndex) ? String(scanner.string[scanLocation]) : nil
} else {
nextChar = nil
}
}
self.performanceLog.tag(with: "(end space)")
return (lastChar, maybeFoundChars, nextChar)
}
func groups( inString string : inout String, forTag tag : String, state : TagState ) -> [TokenGroup] {
var groups : [TokenGroup] = []
var stringList : [String] = []
while string.range(of: tag) != nil {
guard let range = string.range(of: tag) else {
break
}
let beforeTag = String(string[string.startIndex..<range.lowerBound])
if !beforeTag.isEmpty {
stringList.append(beforeTag)
}
let tag = string[range]
stringList.append(String(tag))
string.removeSubrange(string.startIndex..<range.upperBound)
}
if !string.isEmpty {
stringList.append(string)
string.removeAll()
}
if state == .open {
stringList = stringList.reversed()
}
var tagFound = false
for item in stringList {
if item == tag && !tagFound {
var token = TokenGroup(string: item, isEscaped: false, type: .tag)
token.state = state
tagFound = true
groups.append(token)
continue
}
if item == rule.openTag && state == .closed {
var token = TokenGroup(string: item, isEscaped: false, type: .tag)
token.state = .open
groups.append(token)
self.state = .open
continue
}
groups.append(TokenGroup(string: String(item), isEscaped: false, type: .string))
}
return (state == .open) ? groups.reversed() : groups
}
func getTokenGroups( for string : inout String, with rule : CharacterRule, shouldEmpty : Bool = false ) -> [TokenGroup] {
if string.isEmpty {
return []
}
var groups : [TokenGroup] = []
if let closingTag = rule.closingTag, string.contains(closingTag) {
groups.append(contentsOf: self.groups(inString: &string, forTag: closingTag, state: .closed))
} else if let intermediateString = rule.intermediateTag, string.contains(intermediateString) {
groups.append(contentsOf: self.groups(inString: &string, forTag: intermediateString, state: .intermediate))
} else if string.contains(rule.openTag) {
groups.append(contentsOf: self.groups(inString: &string, forTag: rule.openTag, state: .open))
}
if shouldEmpty && !string.isEmpty {
let token = TokenGroup(string: string, isEscaped: false, type: .tag)
groups.append(token)
string.removeAll()
}
return groups
}
/// Checks to ensure that any tags in the rule actually exist in the string.
/// If there are is not at least one of each of the rule's existing tags, there is no processing
/// to be done.
///
/// - Parameter string: The string to check for the existence of the rule's tags.
func verifyTagsExist( _ string : String ) -> Token? {
if !string.contains( rule.openTag ) {
return Token(type: .string, inputString: string)
}
guard let existentClosingTag = rule.closingTag else {
return nil
}
//
if !string.contains(existentClosingTag) {
return Token(type: .string, inputString: string)
}
guard let hasIntermediateString = rule.intermediateTag else {
return nil
}
if !string.contains(hasIntermediateString) {
return Token(type: .string, inputString: string)
}
return nil
}
func scanForRepeatingTags( _ scanner : Scanner, with set : CharacterSet ) -> [Token] {
var tokens : [Token] = []
while !scanner.isAtEnd {
self.performanceLog.tag(with: "(loop start \(rule.openTag))")
tokenGroup += 1
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
if let start = scanner.scanUpToCharacters(from: set) {
self.performanceLog.tag(with: "(first chars \(rule.openTag))")
self.append(start)
}
} else {
var string : NSString?
scanner.scanUpToCharacters(from: set, into: &string)
if let existentString = string as String? {
self.append(existentString)
}
}
// The end of the string
let spacing = self.scanSpacing(scanner, usingCharactersIn: set)
guard let foundTag = spacing.foundChars else {
continue
}
if foundTag == rule.openTag && foundTag.count < rule.minTags {
self.append(foundTag)
continue
}
if !validateSpacing(nextCharacter: spacing.postTag, previousCharacter: spacing.preTag, with: rule) {
let escapeString = String("\(rule.escapeCharacter ?? Character(""))")
let escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
self.append(escaped)
continue
}
self.performanceLog.tag(with: "(found tag \(rule.openTag))")
if !foundTag.contains(rule.openTag) {
self.append(foundTag)
continue
}
var tokenGroups : [TokenGroup] = []
var escapeCharacter : Character? = nil
var cumulatedString = ""
for char in foundTag {
if let existentEscapeCharacter = escapeCharacter {
// If any of the tags feature the current character
let escape = String(existentEscapeCharacter)
let nextTagCharacter = String(char)
if rule.openTag.contains(nextTagCharacter) {
tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
escapeCharacter = nil
} else if nextTagCharacter == escape {
// Doesn't apply to this rule
tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
}
continue
}
if let existentEscape = rule.escapeCharacter {
if char == existentEscape {
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
escapeCharacter = char
continue
}
}
cumulatedString.append(char)
}
if let remainingEscape = escapeCharacter {
tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
}
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
self.append(contentsOf: tokenGroups)
if self.state == .closed {
tokens.append(contentsOf: self.tokens(beginningGroupNumberAt : tokenGroup))
}
}
tokens.append(contentsOf: self.tokens(beginningGroupNumberAt : tokenGroup))
return tokens
}
func scanForTags( _ scanner : Scanner, with set : CharacterSet ) -> [Token] {
var tokens : [Token] = []
while !scanner.isAtEnd {
self.performanceLog.tag(with: "(loop start \(rule.openTag))")
tokenGroup += 1
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
if let start = scanner.scanUpToCharacters(from: set) {
self.performanceLog.tag(with: "(first chars \(rule.openTag))")
self.append(start)
}
} else {
var string : NSString?
scanner.scanUpToCharacters(from: set, into: &string)
if let existentString = string as String? {
self.append(existentString)
}
}
// The end of the string
let spacing = self.scanSpacing(scanner, usingCharactersIn: set)
guard let foundTag = spacing.foundChars else {
continue
}
if foundTag == rule.openTag && foundTag.count < rule.minTags {
self.append(foundTag)
continue
}
if !validateSpacing(nextCharacter: spacing.postTag, previousCharacter: spacing.preTag, with: rule) {
let escapeString = String("\(rule.escapeCharacter ?? Character(""))")
var escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(rule.openTag)", with: rule.openTag)
if let hasIntermediateTag = rule.intermediateTag {
escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(hasIntermediateTag)", with: hasIntermediateTag)
}
if let existentClosingTag = rule.closingTag {
escaped = foundTag.replacingOccurrences(of: "\(escapeString)\(existentClosingTag)", with: existentClosingTag)
}
self.append(escaped)
continue
}
self.performanceLog.tag(with: "(found tag \(rule.openTag))")
if !foundTag.contains(rule.openTag) && !foundTag.contains(rule.intermediateTag ?? "") && !foundTag.contains(rule.closingTag ?? "") {
self.append(foundTag)
continue
}
var tokenGroups : [TokenGroup] = []
var escapeCharacter : Character? = nil
var cumulatedString = ""
for char in foundTag {
if let existentEscapeCharacter = escapeCharacter {
// If any of the tags feature the current character
let escape = String(existentEscapeCharacter)
let nextTagCharacter = String(char)
if rule.openTag.contains(nextTagCharacter) || rule.intermediateTag?.contains(nextTagCharacter) ?? false || rule.closingTag?.contains(nextTagCharacter) ?? false {
tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: true, type: .tag))
escapeCharacter = nil
} else if nextTagCharacter == escape {
// Doesn't apply to this rule
tokenGroups.append(TokenGroup(string: nextTagCharacter, isEscaped: false, type: .escape))
}
continue
}
if let existentEscape = rule.escapeCharacter {
if char == existentEscape {
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
escapeCharacter = char
continue
}
}
cumulatedString.append(char)
}
if let remainingEscape = escapeCharacter {
tokenGroups.append(TokenGroup(string: String(remainingEscape), isEscaped: false, type: .escape))
}
tokenGroups.append(contentsOf: getTokenGroups(for: &cumulatedString, with: rule, shouldEmpty: true))
self.append(contentsOf: tokenGroups)
if self.state == .closed {
tokens.append(contentsOf: self.tokens(beginningGroupNumberAt : tokenGroup))
}
}
tokens.append(contentsOf: self.tokens(beginningGroupNumberAt : tokenGroup))
return tokens
}
func scan( _ string : String, with rule : CharacterRule) -> [Token] {
self.rule = rule
self.tokenGroup = 0
self.performanceLog.start()
if let token = verifyTagsExist(string) {
return [token]
}
let scanner = Scanner(string: string)
scanner.charactersToBeSkipped = nil
var tokens : [Token] = []
let set = CharacterSet(charactersIn: "\(rule.openTag)\(rule.intermediateTag ?? "")\(rule.closingTag ?? "")\(String(rule.escapeCharacter ?? Character.init("") ))")
if rule.isRepeatingTag {
tokens = self.scanForRepeatingTags(scanner, with: set)
} else {
tokens = self.scanForTags(scanner, with: set)
}
self.performanceLog.end()
return tokens
}
func validateSpacing( nextCharacter : String?, previousCharacter : String?, with rule : CharacterRule ) -> Bool {
switch rule.spacesAllowed {
case .leadingSide:
guard nextCharacter != nil else {
return true
}
if nextCharacter == " " {
return false
}
case .trailingSide:
guard previousCharacter != nil else {
return true
}
if previousCharacter == " " {
return false
}
case .no:
switch (previousCharacter, nextCharacter) {
case (nil, nil), ( " ", _ ), ( _, " " ):
return false
default:
return true
}
case .oneSide:
switch (previousCharacter, nextCharacter) {
case (nil, " " ), (" ", nil), (" ", " " ):
return false
default:
return true
}
default:
break
}
return true
}
}
struct TokenGroup {

206
Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift
View File

@ -24,99 +24,108 @@ enum v2_TokenType {
struct v2_Token {
var type : v2_TokenType
let group : Int
let string : String
var metadata : String = ""
// let startIndex : String.Index
}
class SwiftyScannerNonRepeating : SwiftyScanning {
var metadataLookup: [String : String] = [:]
var metadataLookup: [String : String]
var str : String = "" {
didSet {
self.currentIndex = str.startIndex
}
}
var currentIndex : String.Index = "".startIndex
var accumulatedStr : String = ""
var str : String
var currentIndex : String.Index
var rule : CharacterRule
var tokens : [Token]
var openIndices : [Int] = []
var accumulatedStr : String = ""
var stringList : [v2_Token] = []
var rule : CharacterRule! = nil
init() { }
init( tokens : [Token], rule : CharacterRule, metadataLookup : [String : String] = [:] ) {
self.tokens = tokens
self.rule = rule
self.str = tokens.map({ $0.inputString }).joined()
self.currentIndex = self.str.startIndex
self.metadataLookup = metadataLookup
}
func movePointer( _ idx : inout String.Index, addCharacter char : Character? = nil ) {
idx = str.index(idx, offsetBy: 1, limitedBy: str.endIndex) ?? str.endIndex
if let character = char {
accumulatedStr.append(character)
func scan() -> [Token] {
if !self.str.contains(rule.primaryTag.tag) {
return self.tokens
}
self.process()
return self.convertTokens()
}
func emptyAccumulatedString() {
if !accumulatedStr.isEmpty {
stringList.append(v2_Token(type: .string, string: accumulatedStr))
accumulatedStr.removeAll()
}
}
func addLink(with metadataStr : String? = nil) {
let openIndex = openIndices.removeLast()
stringList.remove(at: openIndex)
let subarray = stringList[openIndex..<stringList.count]
stringList.removeSubrange(openIndex..<stringList.count)
stringList.append(v2_Token(type: .link, group: 0, string: subarray.map({ $0.string }).joined(), metadata: metadataStr ?? ""))
}
func scan( _ tokens : [Token], with rule : CharacterRule ) -> [Token] {
}
func scan(_ string: String, with rule: CharacterRule) -> [Token] {
func process() {
var tokens : [Token] = []
self.str = string
self.rule = rule
var isEscape = false
let openTagStart = rule.openTag[rule.openTag.startIndex]
let closeTagStart = ( rule.closeTag != nil ) ? rule.closeTag![rule.closeTag!.startIndex] : nil
let openTagStart = rule.primaryTag.tag[rule.primaryTag.tag.startIndex]
let closeTagStart = ( rule.tag(for: .close)?.tag != nil ) ? rule.tag(for: .close)?.tag[rule.tag(for: .close)!.tag.startIndex] : nil
while currentIndex != str.endIndex {
let char = str[currentIndex]
if char == rule.escapeCharacter {
isEscape = true
movePointer(&currentIndex)
continue
}
if isEscape {
isEscape = false
movePointer(&currentIndex, addCharacter: char)
continue
}
if str[currentIndex] != openTagStart && str[currentIndex] != closeTagStart {
movePointer(&currentIndex, addCharacter: char)
continue
}
if !accumulatedStr.isEmpty {
stringList.append(v2_Token(type: .string, group: 0, string: accumulatedStr))
accumulatedStr.removeAll()
}
// We have the first character of a possible open tag
if char == openTagStart {
guard let nextIdx = str.index(currentIndex, offsetBy: rule.openTag.count, limitedBy: str.endIndex) else {
// Checks to see if there is an escape character before this one
if let prevIndex = str.index(currentIndex, offsetBy: -1, limitedBy: str.startIndex) {
if let escapeChar = self.rule.primaryTag.escapeCharacter(for: str[prevIndex]) {
switch escapeChar.rule {
case .remove:
if !accumulatedStr.isEmpty {
accumulatedStr.removeLast()
}
case .keep:
break
}
movePointer(&currentIndex, addCharacter: char)
continue
}
}
emptyAccumulatedString()
guard let nextIdx = str.index(currentIndex, offsetBy: rule.primaryTag.tag.count, limitedBy: str.endIndex) else {
movePointer(&currentIndex, addCharacter: char)
continue
}
let tag = String(str[currentIndex..<nextIdx])
if tag != rule.openTag {
if tag != rule.primaryTag.tag {
movePointer(&currentIndex, addCharacter: char)
continue
}
openIndices.append(stringList.count)
stringList.append(v2_Token(type: .tag, group: 0, string: tag))
currentIndex = str.index(currentIndex, offsetBy: rule.openTag.count, limitedBy: str.endIndex) ?? str.endIndex
stringList.append(v2_Token(type: .tag, string: tag))
currentIndex = str.index(currentIndex, offsetBy: rule.primaryTag.tag.count, limitedBy: str.endIndex) ?? str.endIndex
continue
}
if char == closeTagStart {
guard let closeTag = rule.closeTag else {
emptyAccumulatedString()
guard let closeTag = rule.tag(for: .close)?.tag else {
movePointer(&currentIndex, addCharacter: char)
continue
}
@ -131,14 +140,14 @@ class SwiftyScannerNonRepeating : SwiftyScanning {
continue
}
if openIndices.isEmpty {
stringList.append(v2_Token(type: .string, group: 0, string: String(char)))
stringList.append(v2_Token(type: .string, string: String(char)))
movePointer(&currentIndex)
continue
}
// At this point we have gathered a valid close tag and we have a valid open tag
guard let metadataOpen = rule.metadataOpen, let close = rule.metadataClose else {
guard let metadataOpen = rule.tag(for: .metadataOpen), let metadataClose = rule.tag(for: .metadataClose) else {
currentIndex = nextIdx
addLink()
continue
@ -147,18 +156,25 @@ class SwiftyScannerNonRepeating : SwiftyScanning {
movePointer(&currentIndex, addCharacter: char)
continue
}
guard str[nextIdx] == rule.metadataOpen else {
guard str[nextIdx] == metadataOpen.tag.first else {
movePointer(&currentIndex, addCharacter: char)
continue
}
let substr = str[nextIdx..<str.endIndex]
guard let closeIdx = substr.firstIndex(of: close) else {
guard let closeIdx = substr.firstIndex(of: metadataClose.tag.first!) else {
movePointer(&currentIndex, addCharacter: char)
continue
}
let open = substr.index(nextIdx, offsetBy: 1, limitedBy: substr.endIndex) ?? substr.endIndex
let metadataStr = String(substr[open..<closeIdx])
guard !metadataStr.contains(rule.primaryTag.tag) else {
movePointer(&currentIndex, addCharacter: char)
continue
}
currentIndex = str.index(closeIdx, offsetBy: 1, limitedBy: str.endIndex) ?? closeIdx
addLink(with: metadataStr)
@ -166,32 +182,68 @@ class SwiftyScannerNonRepeating : SwiftyScanning {
}
if !accumulatedStr.isEmpty {
stringList.append(v2_Token(type: .string, group: 0, string: accumulatedStr))
stringList.append(v2_Token(type: .string, string: accumulatedStr))
}
if !self.stringList.contains(where: { $0.type == .link }) {
return [Token(type: .string, inputString: self.stringList.map({ $0.string}).joined())]
}
func movePointer( _ idx : inout String.Index, addCharacter char : Character? = nil ) {
idx = str.index(idx, offsetBy: 1, limitedBy: str.endIndex) ?? str.endIndex
if let character = char {
accumulatedStr.append(character)
}
for tok in self.stringList {
if tok.type == .string {
tokens.append(Token(type: .string, inputString: tok.string))
}
}
func addLink(with metadataStr : String? = nil) {
let openIndex = openIndices.removeLast()
stringList.remove(at: openIndex)
let subarray = stringList[openIndex..<stringList.count]
stringList.removeSubrange(openIndex..<stringList.count)
stringList.append(v2_Token(type: .link, string: subarray.map({ $0.string }).joined(), metadata: metadataStr ?? ""))
}
func convertTokens() -> [Token] {
if !stringList.contains(where: { $0.type == .link }) {
return [Token(type: .string, inputString: stringList.map({ $0.string}).joined())]
}
var tokens : [Token] = []
var allStrings : [v2_Token] = []
for tok in stringList {
if tok.type == .link {
tokens.append(Token(type: .openTag, inputString: self.rule.openTag))
var token = Token(type: .string, inputString: tok.string, characterStyles: self.rule.styles[1] ?? [])
token.metadataString = tok.metadata
token.isProcessed = true
tokens.append(token)
if let close = self.rule.closeTag {
tokens.append(Token(type: .closeTag, inputString: close))
if !allStrings.isEmpty {
tokens.append(Token(type: .string, inputString: allStrings.map({ $0.string }).joined()))
allStrings.removeAll()
}
let ruleStyles = self.rule.styles[1] ?? []
let charStyles = ( rule.isSelfContained ) ? [] : ruleStyles
var token = Token(type: .string, inputString: tok.string, characterStyles: charStyles)
token.metadataString = tok.metadata
if rule.isSelfContained {
var parentToken = Token(type: .string, inputString: token.id, characterStyles: ruleStyles)
parentToken.children = [token]
tokens.append(parentToken)
} else {
tokens.append(token)
}
} else {
allStrings.append(tok)
}
}
if !allStrings.isEmpty {
tokens.append(Token(type: .string, inputString: allStrings.map({ $0.string }).joined()))
}
return tokens
}
// Old
func scan( _ tokens : [Token], with rule : CharacterRule ) -> [Token] {
self.tokens = tokens
return self.scan(tokens.map({ $0.inputString }).joined(), with: rule)
}
func scan(_ string: String, with rule: CharacterRule) -> [Token] {
return []
}
}

814
Sources/SwiftyMarkdown/SwiftyTokeniser.swift
View File

@ -49,13 +49,15 @@ public class SwiftyTokeniser {
///
/// - Parameter inputString: A string to have the CharacterRules in `self.rules` applied to
public func process( _ inputString : String ) -> [Token] {
var currentTokens = [Token(type: .string, inputString: inputString)]
guard rules.count > 0 else {
return [Token(type: .string, inputString: inputString)]
return currentTokens
}
var currentTokens : [Token] = []
var mutableRules = self.rules
if inputString.isEmpty {
return [Token(type: .string, inputString: "", characterStyles: [])]
}
self.currentPerfomanceLog.start()
@ -64,76 +66,33 @@ public class SwiftyTokeniser {
if nextRule.isRepeatingTag {
self.scanner = SwiftyScanner()
} else {
self.scanner = SwiftyScannerNonRepeating()
self.scanner.metadataLookup = self.metadataLookup
}
self.scanner.metadataLookup = self.metadataLookup
if enableLog {
os_log("------------------------------", log: .tokenising, type: .info)
os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description)
}
self.currentPerfomanceLog.tag(with: "(start rule %@)")
if currentTokens.isEmpty {
// This means it's the first time through
currentTokens = self.applyStyles(to: self.scanner.scan(inputString, with: nextRule), usingRule: nextRule)
continue
}
var outerStringTokens : [Token] = []
var innerStringTokens : [Token] = []
var isOuter = true
for idx in 0..<currentTokens.count {
let nextToken = currentTokens[idx]
if nextToken.type == .openTag && nextToken.isProcessed {
isOuter = false
}
if nextToken.type == .closeTag {
let ref = UUID().uuidString
outerStringTokens.append(Token(type: .replacement, inputString: ref))
innerStringTokens.append(nextToken)
self.replacements[ref] = self.handleReplacementTokens(innerStringTokens, with: nextRule)
innerStringTokens.removeAll()
isOuter = true
continue
}
(isOuter) ? outerStringTokens.append(nextToken) : innerStringTokens.append(nextToken)
}
currentTokens = self.handleReplacementTokens(outerStringTokens, with: nextRule)
var finalTokens : [Token] = []
for token in currentTokens {
guard token.type == .replacement else {
finalTokens.append(token)
continue
}
if let hasReplacement = self.replacements[token.inputString] {
if enableLog {
os_log("Found replacement for %@", log: .tokenising, type: .info, token.inputString)
for (idx,token) in currentTokens.enumerated() {
if !token.children.isEmpty {
if nextRule.isRepeatingTag {
currentTokens[idx].children = self.scanner.scan(token.children, with: nextRule)
} else {
let scanner = SwiftyScannerNonRepeating(tokens: token.children, rule: nextRule, metadataLookup: self.metadataLookup)
currentTokens[idx].children = scanner.scan()
}
for var repToken in hasReplacement {
guard repToken.type == .string else {
finalTokens.append(repToken)
continue
}
for style in token.characterStyles {
if !repToken.characterStyles.contains(where: { $0.isEqualTo(style)}) {
repToken.characterStyles.append(contentsOf: token.characterStyles)
}
}
finalTokens.append(repToken)
}
}
}
currentTokens = finalTokens
// Each string could have additional tokens within it, so they have to be scanned as well with the current rule.
// The one string token might then be exploded into multiple more tokens
if nextRule.isRepeatingTag {
currentTokens = self.scanner.scan(currentTokens, with: nextRule)
} else {
let scanner = SwiftyScannerNonRepeating(tokens: currentTokens, rule: nextRule, metadataLookup: self.metadataLookup)
currentTokens = scanner.scan()
}
}
self.currentPerfomanceLog.tag(with: "(finished all rules)")
@ -142,373 +101,386 @@ public class SwiftyTokeniser {
os_log("=====RULE PROCESSING COMPLETE=====", log: .tokenising, type: .info)
os_log("==================================", log: .tokenising, type: .info)
}
return currentTokens
var output = self.flatten(currentTokens)
return output
}
/// In order to reinsert the original replacements into the new string token, the replacements
/// need to be searched for in the incoming string one by one.
///
/// Using the `newToken(fromSubstring:isReplacement:)` function ensures that any metadata and character styles
/// are passed over into the newly created tokens.
///
/// E.g. A string token that has an `outputString` of "This string AAAAA-BBBBB-CCCCC replacements", with
/// a characterStyle of `bold` for the entire string, needs to be separated into the following tokens:
///
/// - `string`: "This string "
/// - `replacement`: "AAAAA-BBBBB-CCCCC"
/// - `string`: " replacements"
///
/// Each of these need to have a character style of `bold`.
///
/// - Parameters:
/// - replacements: An array of `replacement` tokens
/// - token: The new `string` token that may contain replacement IDs contained in the `replacements` array
func reinsertReplacements(_ replacements : [Token], from stringToken : Token ) -> [Token] {
guard !stringToken.outputString.isEmpty && !replacements.isEmpty else {
return [stringToken]
}
var outputTokens : [Token] = []
let scanner = Scanner(string: stringToken.outputString)
scanner.charactersToBeSkipped = nil
// Remove any replacements that don't appear in the incoming string
var repTokens = replacements.filter({ stringToken.outputString.contains($0.inputString) })
var testString = "\n"
while !scanner.isAtEnd {
var outputString : String = ""
if repTokens.count > 0 {
testString = repTokens.removeFirst().inputString
}
if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
if let nextString = scanner.scanUpToString(testString) {
outputString = nextString
outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
if let outputToken = scanner.scanString(testString) {
outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
}
} else if let outputToken = scanner.scanString(testString) {
outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
}
func flatten( _ tokens : [Token], with styles : [CharacterStyling] = []) -> [Token] {
var output : [Token] = []
for var token in tokens {
if !token.children.isEmpty {
output.append(contentsOf: self.flatten(token.children, with: token.characterStyles))
} else {
var oldString : NSString? = nil
var tokenString : NSString? = nil
scanner.scanUpTo(testString, into: &oldString)
if let nextString = oldString {
outputString = nextString as String
outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
scanner.scanString(testString, into: &tokenString)
if let outputToken = tokenString as String? {
outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
}
} else {
scanner.scanString(testString, into: &tokenString)
if let outputToken = tokenString as String? {
outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
}
}
token.characterStyles.append(contentsOf: styles)
output.append(token)
}
}
return outputTokens
return output
}
/// This function is necessary because a previously tokenised string might have
///
/// Consider a previously tokenised string, where AAAAA-BBBBB-CCCCC represents a replaced \[link\](url) instance.
///
/// The incoming tokens will look like this:
///
/// - `string`: "A \*\*Bold"
/// - `replacement` : "AAAAA-BBBBB-CCCCC"
/// - `string`: " with a trailing string**"
///
/// However, because the scanner can only tokenise individual strings, passing in the string values
/// of these tokens individually and applying the styles will not correctly detect the starting and
/// ending `repeatingTag` instances. (e.g. the scanner will see "A \*\*Bold", and then "AAAAA-BBBBB-CCCCC",
/// and finally " with a trailing string\*\*")
///
/// The strings need to be combined, so that they form a single string:
/// A \*\*Bold AAAAA-BBBBB-CCCCC with a trailing string\*\*.
/// This string is then parsed and tokenised so that it looks like this:
///
/// - `string`: "A "
/// - `repeatingTag`: "\*\*"
/// - `string`: "Bold AAAAA-BBBBB-CCCCC with a trailing string"
/// - `repeatingTag`: "\*\*"
///
/// Finally, the replacements from the original incoming token array are searched for and pulled out
/// of this new string, so the final result looks like this:
///
/// - `string`: "A "
/// - `repeatingTag`: "\*\*"
/// - `string`: "Bold "
/// - `replacement`: "AAAAA-BBBBB-CCCCC"
/// - `string`: " with a trailing string"
/// - `repeatingTag`: "\*\*"
///
/// - Parameters:
/// - tokens: The tokens to be combined, scanned, re-tokenised, and merged
/// - rule: The character rule currently being applied
func scanReplacementTokens( _ tokens : [Token], with rule : CharacterRule ) -> [Token] {
guard tokens.count > 0 else {
return []
}
let combinedString = tokens.map({ $0.outputString }).joined()
let nextTokens = self.scanner.scan(combinedString, with: rule)
var replacedTokens = self.applyStyles(to: nextTokens, usingRule: rule)
/// It's necessary here to check to see if the first token (which will always represent the styles
/// to be applied from previous scans) has any existing metadata or character styles and apply them
/// to *all* the string and replacement tokens found by the new scan.
for idx in 0..<replacedTokens.count {
guard replacedTokens[idx].type == .string || replacedTokens[idx].type == .replacement else {
continue
}
if tokens.first!.metadataString != nil && replacedTokens[idx].metadataString == nil {
replacedTokens[idx].metadataString = tokens.first!.metadataString
}
replacedTokens[idx].characterStyles.append(contentsOf: tokens.first!.characterStyles)
}
// Swap the original replacement tokens back in
let replacements = tokens.filter({ $0.type == .replacement })
var outputTokens : [Token] = []
for token in replacedTokens {
guard token.type == .string else {
outputTokens.append(token)
continue
}
outputTokens.append(contentsOf: self.reinsertReplacements(replacements, from: token))
}
return outputTokens
}
/// This function ensures that only concurrent `string` and `replacement` tokens are processed together.
///
/// i.e. If there is an existing `repeatingTag` token between two strings, then those strings will be
/// processed individually. This prevents incorrect parsing of strings like "\*\*\_Should only be bold\*\*\_"
///
/// - Parameters:
/// - incomingTokens: A group of tokens whose string tokens and replacement tokens should be combined and re-tokenised
/// - rule: The current rule being processed
func handleReplacementTokens( _ incomingTokens : [Token], with rule : CharacterRule) -> [Token] {
// Only combine string and replacements that are next to each other.
var newTokenSet : [Token] = []
var currentTokenSet : [Token] = []
for i in 0..<incomingTokens.count {
guard incomingTokens[i].type == .string || incomingTokens[i].type == .replacement else {
newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
newTokenSet.append(incomingTokens[i])
currentTokenSet.removeAll()
continue
}
guard !incomingTokens[i].isProcessed && !incomingTokens[i].isMetadata && !incomingTokens[i].shouldSkip else {
newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
newTokenSet.append(incomingTokens[i])
currentTokenSet.removeAll()
continue
}
currentTokenSet.append(incomingTokens[i])
}
newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
return newTokenSet
}
func handleClosingTagFromOpenTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule ) {
guard rule.closingTag != nil else {
return
}
guard let closeTokenIdx = tokens.firstIndex(where: { $0.type == .closeTag && !$0.isProcessed }) else {
return
}
var metadataIndex = index
// If there's an intermediate tag, get the index of that
if rule.intermediateTag != nil {
guard let nextTokenIdx = tokens.firstIndex(where: { $0.type == .intermediateTag && !$0.isProcessed }) else {
return
}
metadataIndex = nextTokenIdx
let styles : [CharacterStyling] = rule.styles[1] ?? []
for i in index..<nextTokenIdx {
for style in styles {
if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
tokens[i].characterStyles.append(style)
}
}
}
}
var metadataString : String = ""
for i in metadataIndex..<closeTokenIdx {
if tokens[i].type == .string {
metadataString.append(tokens[i].outputString)
tokens[i].isMetadata = true
}
}
for i in index..<metadataIndex {
if tokens[i].type == .string {
tokens[i].metadataString = metadataString
}
}
tokens[closeTokenIdx].isProcessed = true
tokens[metadataIndex].isProcessed = true
tokens[index].isProcessed = true
}
/// This is here to manage how opening tags are matched with closing tags when they're all the same
/// character.
///
/// Of course, because Markdown is about as loose as a spec can be while still being considered any
/// kind of spec, the number of times this character repeats causes different effects. Then there
/// is the ill-defined way it should work if the number of opening and closing tags are different.
///
/// - Parameters:
/// - index: The index of the current token in the loop
/// - tokens: An inout variable of the loop tokens of interest
/// - rule: The character rule being applied
func handleClosingTagFromRepeatingTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule) {
let theToken = tokens[index]
if enableLog {
os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
}
guard theToken.count > 0 else {
return
}
let startIdx = index
var endIdx : Int? = nil
let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
// Try to find exact match first
if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) {
endIdx = nextTokenIdx
}
if endIdx == nil, let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count >= 1 && $0.id != theToken.id && !$0.isProcessed }) {
endIdx = nextTokenIdx
}
guard let existentEnd = endIdx else {
return
}
let styles : [CharacterStyling] = rule.styles[maxCount] ?? []
for i in startIdx..<existentEnd {
for style in styles {
if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
tokens[i].characterStyles.append(style)
}
}
if rule.cancels == .allRemaining {
tokens[i].shouldSkip = true
}
}
let maxEnd = (tokens[existentEnd].count > rule.maxTags) ? rule.maxTags : tokens[existentEnd].count
tokens[index].count = theToken.count - maxEnd
tokens[existentEnd].count = tokens[existentEnd].count - maxEnd
if maxEnd < rule.maxTags {
self.handleClosingTagFromRepeatingTag(withIndex: index, in: &tokens, following: rule)
} else {
tokens[existentEnd].isProcessed = true
tokens[index].isProcessed = true
}
}
func applyStyles( to tokens : [Token], usingRule rule : CharacterRule ) -> [Token] {
var mutableTokens : [Token] = tokens
if enableLog {
os_log("Applying styles to tokens: %@", log: .tokenising, type: .info, tokens.oslogDisplay )
}
for idx in 0..<mutableTokens.count {
let token = mutableTokens[idx]
switch token.type {
case .escape:
if enableLog {
os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
}
case .repeatingTag:
let theToken = mutableTokens[idx]
self.handleClosingTagFromRepeatingTag(withIndex: idx, in: &mutableTokens, following: rule)
if enableLog {
os_log("Found repeating tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
}
case .openTag:
let theToken = mutableTokens[idx]
if enableLog {
os_log("Found open tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
}
guard rule.closingTag != nil else {
// If there's an intermediate tag, get the index of that
// Get the index of the closing tag
continue
}
self.handleClosingTagFromOpenTag(withIndex: idx, in: &mutableTokens, following: rule)
case .intermediateTag:
let theToken = mutableTokens[idx]
if enableLog {
os_log("Found intermediate tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
}
case .closeTag:
let theToken = mutableTokens[idx]
if enableLog {
os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
}
case .string:
let theToken = mutableTokens[idx]
if enableLog {
if theToken.isMetadata {
os_log("Found Metadata: %@", log: .tokenising, type: .info, theToken.inputString )
} else {
os_log("Found String: %@", log: .tokenising, type: .info, theToken.inputString )
}
if let hasMetadata = theToken.metadataString {
os_log("...with metadata: %@", log: .tokenising, type: .info, hasMetadata )
}
}
case .replacement:
if enableLog {
os_log("Found replacement with ID: %@", log: .tokenising, type: .info, mutableTokens[idx].inputString )
}
}
}
return mutableTokens
}
//
//
// /// In order to reinsert the original replacements into the new string token, the replacements
// /// need to be searched for in the incoming string one by one.
// ///
// /// Using the `newToken(fromSubstring:isReplacement:)` function ensures that any metadata and character styles
// /// are passed over into the newly created tokens.
// ///
// /// E.g. A string token that has an `outputString` of "This string AAAAA-BBBBB-CCCCC replacements", with
// /// a characterStyle of `bold` for the entire string, needs to be separated into the following tokens:
// ///
// /// - `string`: "This string "
// /// - `replacement`: "AAAAA-BBBBB-CCCCC"
// /// - `string`: " replacements"
// ///
// /// Each of these need to have a character style of `bold`.
// ///
// /// - Parameters:
// /// - replacements: An array of `replacement` tokens
// /// - token: The new `string` token that may contain replacement IDs contained in the `replacements` array
// func reinsertReplacements(_ replacements : [Token], from stringToken : Token ) -> [Token] {
// guard !stringToken.outputString.isEmpty && !replacements.isEmpty else {
// return [stringToken]
// }
// var outputTokens : [Token] = []
// let scanner = Scanner(string: stringToken.outputString)
// scanner.charactersToBeSkipped = nil
//
// // Remove any replacements that don't appear in the incoming string
// var repTokens = replacements.filter({ stringToken.outputString.contains($0.inputString) })
//
// var testString = "\n"
// while !scanner.isAtEnd {
// var outputString : String = ""
// if repTokens.count > 0 {
// testString = repTokens.removeFirst().inputString
// }
//
// if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
// if let nextString = scanner.scanUpToString(testString) {
// outputString = nextString
// outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
// if let outputToken = scanner.scanString(testString) {
// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
// }
// } else if let outputToken = scanner.scanString(testString) {
// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
// }
// } else {
// var oldString : NSString? = nil
// var tokenString : NSString? = nil
// scanner.scanUpTo(testString, into: &oldString)
// if let nextString = oldString {
// outputString = nextString as String
// outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
// scanner.scanString(testString, into: &tokenString)
// if let outputToken = tokenString as String? {
// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
// }
// } else {
// scanner.scanString(testString, into: &tokenString)
// if let outputToken = tokenString as String? {
// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
// }
// }
// }
// }
// return outputTokens
// }
//
//
// /// This function is necessary because a previously tokenised string might have
// ///
// /// Consider a previously tokenised string, where AAAAA-BBBBB-CCCCC represents a replaced \[link\](url) instance.
// ///
// /// The incoming tokens will look like this:
// ///
// /// - `string`: "A \*\*Bold"
// /// - `replacement` : "AAAAA-BBBBB-CCCCC"
// /// - `string`: " with a trailing string**"
// ///
// /// However, because the scanner can only tokenise individual strings, passing in the string values
// /// of these tokens individually and applying the styles will not correctly detect the starting and
// /// ending `repeatingTag` instances. (e.g. the scanner will see "A \*\*Bold", and then "AAAAA-BBBBB-CCCCC",
// /// and finally " with a trailing string\*\*")
// ///
// /// The strings need to be combined, so that they form a single string:
// /// A \*\*Bold AAAAA-BBBBB-CCCCC with a trailing string\*\*.
// /// This string is then parsed and tokenised so that it looks like this:
// ///
// /// - `string`: "A "
// /// - `repeatingTag`: "\*\*"
// /// - `string`: "Bold AAAAA-BBBBB-CCCCC with a trailing string"
// /// - `repeatingTag`: "\*\*"
// ///
// /// Finally, the replacements from the original incoming token array are searched for and pulled out
// /// of this new string, so the final result looks like this:
// ///
// /// - `string`: "A "
// /// - `repeatingTag`: "\*\*"
// /// - `string`: "Bold "
// /// - `replacement`: "AAAAA-BBBBB-CCCCC"
// /// - `string`: " with a trailing string"
// /// - `repeatingTag`: "\*\*"
// ///
// /// - Parameters:
// /// - tokens: The tokens to be combined, scanned, re-tokenised, and merged
// /// - rule: The character rule currently being applied
// func scanReplacementTokens( _ tokens : [Token], with rule : CharacterRule ) -> [Token] {
// guard tokens.count > 0 else {
// return []
// }
//
// let combinedString = tokens.map({ $0.outputString }).joined()
//
// let nextTokens = self.scanner.scan(combinedString, with: rule)
// var replacedTokens = self.applyStyles(to: nextTokens, usingRule: rule)
//
// /// It's necessary here to check to see if the first token (which will always represent the styles
// /// to be applied from previous scans) has any existing metadata or character styles and apply them
// /// to *all* the string and replacement tokens found by the new scan.
// for idx in 0..<replacedTokens.count {
// guard replacedTokens[idx].type == .string || replacedTokens[idx].type == .replacement else {
// continue
// }
// if tokens.first!.metadataString != nil && replacedTokens[idx].metadataString == nil {
// replacedTokens[idx].metadataString = tokens.first!.metadataString
// }
// replacedTokens[idx].characterStyles.append(contentsOf: tokens.first!.characterStyles)
// }
//
// // Swap the original replacement tokens back in
// let replacements = tokens.filter({ $0.type == .replacement })
// var outputTokens : [Token] = []
// for token in replacedTokens {
// guard token.type == .string else {
// outputTokens.append(token)
// continue
// }
// outputTokens.append(contentsOf: self.reinsertReplacements(replacements, from: token))
// }
//
// return outputTokens
// }
//
//
//
// /// This function ensures that only concurrent `string` and `replacement` tokens are processed together.
// ///
// /// i.e. If there is an existing `repeatingTag` token between two strings, then those strings will be
// /// processed individually. This prevents incorrect parsing of strings like "\*\*\_Should only be bold\*\*\_"
// ///
// /// - Parameters:
// /// - incomingTokens: A group of tokens whose string tokens and replacement tokens should be combined and re-tokenised
// /// - rule: The current rule being processed
// func handleReplacementTokens( _ incomingTokens : [Token], with rule : CharacterRule) -> [Token] {
//
// // Only combine string and replacements that are next to each other.
// var newTokenSet : [Token] = []
// var currentTokenSet : [Token] = []
// for i in 0..<incomingTokens.count {
// guard incomingTokens[i].type == .string || incomingTokens[i].type == .replacement else {
// newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
// newTokenSet.append(incomingTokens[i])
// currentTokenSet.removeAll()
// continue
// }
// guard !incomingTokens[i].isProcessed && !incomingTokens[i].isMetadata && !incomingTokens[i].shouldSkip else {
// newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
// newTokenSet.append(incomingTokens[i])
// currentTokenSet.removeAll()
// continue
// }
// currentTokenSet.append(incomingTokens[i])
// }
// newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
//
// return newTokenSet
// }
//
//
// func handleClosingTagFromOpenTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule ) {
//
// guard rule.closeTag != nil else {
// return
// }
// guard let closeTokenIdx = tokens.firstIndex(where: { $0.type == .closeTag && !$0.isProcessed }) else {
// return
// }
//
// var metadataIndex = index
// // If there's an intermediate tag, get the index of that
// if rule.intermediateTag != nil {
// guard let nextTokenIdx = tokens.firstIndex(where: { $0.type == .intermediateTag && !$0.isProcessed }) else {
// return
// }
// metadataIndex = nextTokenIdx
// let styles : [CharacterStyling] = rule.styles[1] ?? []
// for i in index..<nextTokenIdx {
// for style in styles {
// if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
// tokens[i].characterStyles.append(style)
// }
// }
// }
// }
//
// var metadataString : String = ""
// for i in metadataIndex..<closeTokenIdx {
// if tokens[i].type == .string {
// metadataString.append(tokens[i].outputString)
// tokens[i].isMetadata = true
// }
// }
//
// for i in index..<metadataIndex {
// if tokens[i].type == .string {
// tokens[i].metadataString = metadataString
// }
// }
//
// tokens[closeTokenIdx].isProcessed = true
// tokens[metadataIndex].isProcessed = true
// tokens[index].isProcessed = true
// }
//
//
// /// This is here to manage how opening tags are matched with closing tags when they're all the same
// /// character.
// ///
// /// Of course, because Markdown is about as loose as a spec can be while still being considered any
// /// kind of spec, the number of times this character repeats causes different effects. Then there
// /// is the ill-defined way it should work if the number of opening and closing tags are different.
// ///
// /// - Parameters:
// /// - index: The index of the current token in the loop
// /// - tokens: An inout variable of the loop tokens of interest
// /// - rule: The character rule being applied
// func handleClosingTagFromRepeatingTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule) {
// let theToken = tokens[index]
//
// if enableLog {
// os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
// }
//
// guard theToken.count > 0 else {
// return
// }
//
// let startIdx = index
// var endIdx : Int? = nil
//
// let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
// // Try to find exact match first
// if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) {
// endIdx = nextTokenIdx
// }
//
// if endIdx == nil, let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count >= 1 && $0.id != theToken.id && !$0.isProcessed }) {
// endIdx = nextTokenIdx
// }
// guard let existentEnd = endIdx else {
// return
// }
//
//
// let styles : [CharacterStyling] = rule.styles[maxCount] ?? []
// for i in startIdx..<existentEnd {
// for style in styles {
// if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
// tokens[i].characterStyles.append(style)
// }
// }
// if rule.cancels == .allRemaining {
// tokens[i].shouldSkip = true
// }
// }
//
// let maxEnd = (tokens[existentEnd].count > rule.maxTags) ? rule.maxTags : tokens[existentEnd].count
// tokens[index].count = theToken.count - maxEnd
// tokens[existentEnd].count = tokens[existentEnd].count - maxEnd
// if maxEnd < rule.maxTags {
// self.handleClosingTagFromRepeatingTag(withIndex: index, in: &tokens, following: rule)
// } else {
// tokens[existentEnd].isProcessed = true
// tokens[index].isProcessed = true
// }
//
//
// }
//
// func applyStyles( to tokens : [Token], usingRule rule : CharacterRule ) -> [Token] {
// var mutableTokens : [Token] = tokens
//
// if enableLog {
// os_log("Applying styles to tokens: %@", log: .tokenising, type: .info, tokens.oslogDisplay )
// }
// for idx in 0..<mutableTokens.count {
// let token = mutableTokens[idx]
// switch token.type {
// case .escape:
// if enableLog {
// os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
// }
// case .repeatingTag:
// let theToken = mutableTokens[idx]
// self.handleClosingTagFromRepeatingTag(withIndex: idx, in: &mutableTokens, following: rule)
// if enableLog {
// os_log("Found repeating tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
// }
// case .openTag:
// let theToken = mutableTokens[idx]
// if enableLog {
// os_log("Found open tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
// }
//
// guard rule.closingTag != nil else {
//
// // If there's an intermediate tag, get the index of that
//
// // Get the index of the closing tag
//
// continue
// }
// self.handleClosingTagFromOpenTag(withIndex: idx, in: &mutableTokens, following: rule)
//
//
// case .intermediateTag:
// let theToken = mutableTokens[idx]
// if enableLog {
// os_log("Found intermediate tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
// }
//
// case .closeTag:
// let theToken = mutableTokens[idx]
// if enableLog {
// os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
// }
//
// case .string:
// let theToken = mutableTokens[idx]
// if enableLog {
// if theToken.isMetadata {
// os_log("Found Metadata: %@", log: .tokenising, type: .info, theToken.inputString )
// } else {
// os_log("Found String: %@", log: .tokenising, type: .info, theToken.inputString )
// }
// if let hasMetadata = theToken.metadataString {
// os_log("...with metadata: %@", log: .tokenising, type: .info, hasMetadata )
// }
// }
//
// case .replacement:
// if enableLog {
// os_log("Found replacement with ID: %@", log: .tokenising, type: .info, mutableTokens[idx].inputString )
// }
// }
// }
// return mutableTokens
// }
//
//
//
}

24
Tests/SwiftyMarkdownTests/SwiftyMarkdownCharacterTests.swift
View File

@ -13,6 +13,30 @@ class SwiftyMarkdownStylingTests: SwiftyMarkdownCharacterTests {
func testIsolatedCase() {
challenge = TokenTest(input: "An ![Image](imageName)", output: "An ", tokens: [
Token(type: .string, inputString: "An ", characterStyles: []),
Token(type: .string, inputString: "Image", characterStyles: [CharacterStyle.image])
])
results = self.attempt(challenge, rules: [.links, .images])
if results.stringTokens.count == challenge.tokens.count {
for (idx, token) in results.stringTokens.enumerated() {
XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as? [CharacterStyle])
}
} else {
XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
}
XCTAssertEqual(results.attributedString.string, challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.image) ?? false) })
if links.count == 1 {
XCTAssertEqual(links[0].metadataString, "imageName")
} else {
XCTFail("Incorrect link count. Expecting 1, found \(links.count)")
}
return
challenge = TokenTest(input: """
An asterisk: *
Line break

60
Tests/SwiftyMarkdownTests/SwiftyMarkdownLinkTests.swift
View File

@ -12,8 +12,8 @@ import XCTest
class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
func testSingleLinkPositions() {
challenge = TokenTest(input: "[Link](http://voyagetravelapps.com/)", output: "Link", tokens: [
Token(type: .string, inputString: "Link", characterStyles: [CharacterStyle.link])
challenge = TokenTest(input: "[a](b)", output: "a", tokens: [
Token(type: .string, inputString: "a", characterStyles: [CharacterStyle.link])
])
results = self.attempt(challenge)
if results.stringTokens.count == challenge.tokens.count {
@ -27,7 +27,7 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
if let existentOpen = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.link) ?? false) }).first {
XCTAssertEqual(existentOpen.metadataString, "http://voyagetravelapps.com/")
XCTAssertEqual(existentOpen.metadataString, "b")
} else {
XCTFail("Failed to find an open link tag")
}
@ -87,6 +87,38 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
XCTAssertEqual(results.attributedString.string, challenge.output)
}
func testEscapedLinks() {
challenge = TokenTest(input: "\\[a](b)", output: "[a](b)", tokens: [
Token(type: .string, inputString: "[a](b)", characterStyles: [])
])
results = self.attempt(challenge, rules: [.images, .referencedLinks, .links])
if results.stringTokens.count == challenge.tokens.count {
for (idx, token) in results.stringTokens.enumerated() {
XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as? [CharacterStyle])
}
} else {
XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
}
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
challenge = TokenTest(input: "![a](b)", output: "![a](b)", tokens: [
Token(type: .string, inputString: "![a](b)", characterStyles: [])
])
results = self.attempt(challenge, rules: [.links])
if results.stringTokens.count == challenge.tokens.count {
for (idx, token) in results.stringTokens.enumerated() {
XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
XCTAssertEqual(token.characterStyles as? [CharacterStyle], challenge.tokens[idx].characterStyles as? [CharacterStyle])
}
} else {
XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
}
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
}
func testMultipleLinkPositions() {
challenge = TokenTest(input: "[Link 1](http://voyagetravelapps.com/)[Link 2](https://www.neverendingvoyage.com/)", output: "Link 1Link 2", tokens: [
@ -164,7 +196,7 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
XCTFail("Incorrect number of links found. Expecting 2, found \(links.count)")
}
challenge = TokenTest(input: "String at start [Link 1](http://voyagetravelapps.com/)[Link 2](https://www.neverendingvoyage.com/)", output: "Link 1, Link 2", tokens: [
challenge = TokenTest(input: "String at start [Link 1](http://voyagetravelapps.com/)[Link 2](https://www.neverendingvoyage.com/)", output: "String at start Link 1Link 2", tokens: [
Token(type: .string, inputString: "String at start ", characterStyles: []),
Token(type: .string, inputString: "Link 1", characterStyles: [CharacterStyle.link]),
Token(type: .string, inputString: "Link 2", characterStyles: [CharacterStyle.link])
@ -378,7 +410,8 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
func testMalformedLinksWithValidLinks() {
challenge = TokenTest(input: "[Link with missing parenthesis](http://voyagetravelapps.com/ followed by a [valid link](http://voyagetravelapps.com/)", output: "[Link with missing parenthesis](http://voyagetravelapps.com/ followed by a valid link", tokens: [
Token(type: .string, inputString: "[Link with missing parenthesis](http://voyagetravelapps.com/", characterStyles: [])
Token(type: .string, inputString: "[Link with missing parenthesis](http://voyagetravelapps.com/ followed by a ", characterStyles: []),
Token(type: .string, inputString: "valid link", characterStyles: [CharacterStyle.link])
])
results = self.attempt(challenge)
XCTAssertEqual(results.stringTokens.count, challenge.tokens.count )
@ -391,8 +424,9 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
XCTFail("Incorrect link count. Expecting 1, found \(results.links.count)")
}
challenge = TokenTest(input: "A [Link](http://voyagetravelapps.com/ followed by a [valid link](http://voyagetravelapps.com/)", output: "A [Link](http://voyagetravelapps.com/", tokens: [
Token(type: .string, inputString: "A [Link](http://voyagetravelapps.com/ followed by a valid link", characterStyles: [])
challenge = TokenTest(input: "A [Link](http://voyagetravelapps.com/ followed by a [valid link](http://voyagetravelapps.com/)", output: "A [Link](http://voyagetravelapps.com/ followed by a valid link", tokens: [
Token(type: .string, inputString: "A [Link](http://voyagetravelapps.com/ followed by a ", characterStyles: []),
Token(type: .string, inputString: "valid link", characterStyles: [CharacterStyle.link])
])
results = self.attempt(challenge)
if results.stringTokens.count == challenge.tokens.count {
@ -413,7 +447,7 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
}
challenge = TokenTest(input: "[Link with missing square(http://voyagetravelapps.com/) followed by a [valid link](http://voyagetravelapps.com/)", output: "[Link with missing square(http://voyagetravelapps.com/) followed by a valid link", tokens: [
Token(type: .string, inputString: "Link with missing square(http://voyagetravelapps.com/) followed by a ", characterStyles: []),
Token(type: .string, inputString: "[Link with missing square(http://voyagetravelapps.com/) followed by a ", characterStyles: []),
Token(type: .string, inputString: "valid link", characterStyles: [CharacterStyle.link])
])
results = self.attempt(challenge)
@ -435,7 +469,8 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
}
challenge = TokenTest(input: "A [Link(http://voyagetravelapps.com/) followed by a [valid link](http://voyagetravelapps.com/)", output: "A [Link(http://voyagetravelapps.com/) followed by a valid link", tokens: [
Token(type: .string, inputString: "A [Link(http://voyagetravelapps.com/)", characterStyles: [])
Token(type: .string, inputString: "A [Link(http://voyagetravelapps.com/) followed by a ", characterStyles: []),
Token(type: .string, inputString: "valid link", characterStyles: [CharacterStyle.link])
])
results = self.attempt(challenge)
if results.stringTokens.count == challenge.tokens.count {
@ -518,11 +553,11 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
}
func testForImages() {
challenge = TokenTest(input: "An ![Image](imageName)", output: "An Image", tokens: [
challenge = TokenTest(input: "An ![Image](imageName)", output: "An ", tokens: [
Token(type: .string, inputString: "An ", characterStyles: []),
Token(type: .string, inputString: "Image", characterStyles: [CharacterStyle.image])
])
results = self.attempt(challenge)
results = self.attempt(challenge, rules: [.links, .images])
if results.stringTokens.count == challenge.tokens.count {
for (idx, token) in results.stringTokens.enumerated() {
XCTAssertEqual(token.inputString, challenge.tokens[idx].inputString)
@ -531,6 +566,7 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
} else {
XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
}
XCTAssertEqual(results.attributedString.string, challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
var links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.image) ?? false) })
if links.count == 1 {
@ -552,6 +588,7 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
} else {
XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
}
XCTAssertEqual(results.attributedString.string, challenge.output)
XCTAssertEqual(results.foundStyles, results.expectedStyles)
links = results.tokens.filter({ $0.type == .string && (($0.characterStyles as? [CharacterStyle])?.contains(.image) ?? false) })
if links.count == 1 {
@ -582,6 +619,7 @@ class SwiftyMarkdownLinkTests: SwiftyMarkdownCharacterTests {
XCTAssertEqual(results.stringTokens.count, challenge.tokens.count)
}
XCTAssertEqual(results.foundStyles, results.expectedStyles)
XCTAssertEqual(results.attributedString.string, challenge.output)
if results.links.count == 1 {
XCTAssertEqual(results.links[0].metadataString, "https://www.neverendingvoyage.com/")
} else {

27
Tests/SwiftyMarkdownTests/XCTest+SwiftyMarkdown.swift
View File

@ -31,19 +31,31 @@ enum Rule {
func asCharacterRule() -> CharacterRule {
switch self {
case .images:
return CharacterRule(openTag: "![", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.image]], maxTags: 1)
return CharacterRule(primaryTag: CharacterRuleTag(tag: "![", type: .open), otherTags: [
CharacterRuleTag(tag: "]", type: .close),
CharacterRuleTag(tag: "(", type: .metadataOpen),
CharacterRuleTag(tag: ")", type: .metadataClose)
], styles: [1 : [CharacterStyle.image]], metadataLookup: false, spacesAllowed: .bothSides, isSelfContained: true)
case .links:
return CharacterRule(openTag: "[", intermediateTag: "](", closingTag: ")", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], maxTags: 1)
return CharacterRule(primaryTag: CharacterRuleTag(tag: "[", type: .open, escapeCharacters: [EscapeCharacter(character: "\\", rule: .remove),EscapeCharacter(character: "!", rule: .keep)]), otherTags: [
CharacterRuleTag(tag: "]", type: .close),
CharacterRuleTag(tag: "(", type: .metadataOpen),
CharacterRuleTag(tag: ")", type: .metadataClose)
], styles: [1 : [CharacterStyle.link]], metadataLookup: true, spacesAllowed: .bothSides, isSelfContained: true)
case .backticks:
return CharacterRule(openTag: "`", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.code]], maxTags: 1, cancels: .allRemaining)
return CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : [CharacterStyle.code]], cancels: .allRemaining)
case .strikethroughs:
return CharacterRule(openTag: "~", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [2 : [CharacterStyle.strikethrough]], minTags: 2, maxTags: 2)
return CharacterRule(primaryTag:CharacterRuleTag(tag: "~", type: .repeating, min: 2, max: 2), otherTags : [], styles: [2 : [CharacterStyle.strikethrough]])
case .asterisks:
return CharacterRule(openTag: "*", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3)
return CharacterRule(primaryTag: CharacterRuleTag(tag: "*", type: .repeating, min: 1, max: 3), otherTags: [], styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]])
case .underscores:
return CharacterRule(openTag: "_", intermediateTag: nil, closingTag: nil, escapeCharacter: "\\", styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]], maxTags: 3)
return CharacterRule(primaryTag: CharacterRuleTag(tag: "_", type: .repeating, min: 1, max: 3), otherTags: [], styles: [1 : [CharacterStyle.italic], 2 : [CharacterStyle.bold], 3 : [CharacterStyle.bold, CharacterStyle.italic]])
case .referencedLinks:
return CharacterRule(openTag: "[", intermediateTag: "][", closingTag: "]", escapeCharacter: "\\", styles: [1 : [CharacterStyle.link]], maxTags: 1)
return CharacterRule(primaryTag: CharacterRuleTag(tag: "[", type: .open, escapeCharacters: [EscapeCharacter(character: "\\", rule: .remove),EscapeCharacter(character: "!", rule: .keep)]), otherTags: [
CharacterRuleTag(tag: "]", type: .close),
CharacterRuleTag(tag: "[", type: .metadataOpen),
CharacterRuleTag(tag: "]", type: .metadataClose)
], styles: [1 : [CharacterStyle.referencedLink]], metadataLookup: true, spacesAllowed: .bothSides, isSelfContained: true)
}
}
@ -63,6 +75,7 @@ class SwiftyMarkdownCharacterTests : XCTestCase {
}
let md = SwiftyMarkdown(string: challenge.input)
md.applyAttachments = false
let attributedString = md.attributedString()
let tokens : [Token] = md.previouslyFoundTokens
let stringTokens = tokens.filter({ $0.type == .string && !$0.isMetadata })