From 61490b1e135b663d5803e1b6f55d31b94f86729f Mon Sep 17 00:00:00 2001 From: Simon Fairbairn Date: Sat, 11 Apr 2020 16:08:02 +1200 Subject: [PATCH] Cleans up codebase and class names --- .../project.pbxproj | 10 +- Example/SwiftyMarkdownExample/example copy.md | 50 -- Example/SwiftyMarkdownExample/example.md | 51 +- Sources/SwiftyMarkdown/SwiftyMarkdown.swift | 2 +- Sources/SwiftyMarkdown/SwiftyScanner.swift | 567 +++++++++++++++++- .../SwiftyScannerNonRepeating.swift | 565 ----------------- Sources/SwiftyMarkdown/SwiftyTokeniser.swift | 378 +----------- 7 files changed, 593 insertions(+), 1030 deletions(-) delete mode 100644 Example/SwiftyMarkdownExample/example copy.md delete mode 100644 Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift diff --git a/Example/SwiftyMarkdownExample.xcodeproj/project.pbxproj b/Example/SwiftyMarkdownExample.xcodeproj/project.pbxproj index 6e93a9e..6c5cadc 100644 --- a/Example/SwiftyMarkdownExample.xcodeproj/project.pbxproj +++ b/Example/SwiftyMarkdownExample.xcodeproj/project.pbxproj @@ -7,7 +7,6 @@ objects = { /* Begin PBXBuildFile section */ - F421DD991C8AF4E900B86D66 /* example.md in Resources */ = {isa = PBXBuildFile; fileRef = F421DD951C8AF34F00B86D66 /* example.md */; }; F4B4A44C23E4E17400550249 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4B4A44B23E4E17400550249 /* AppDelegate.swift */; }; F4B4A44E23E4E17400550249 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4B4A44D23E4E17400550249 /* ViewController.swift */; }; F4B4A45023E4E17400550249 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = F4B4A44F23E4E17400550249 /* Assets.xcassets */; }; @@ -20,6 +19,7 @@ F4CE98B61C8AEF7D00D735C1 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = F4CE98B41C8AEF7D00D735C1 /* LaunchScreen.storyboard */; }; F4CE98C11C8AEF7D00D735C1 /* SwiftyMarkdownExampleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4CE98C01C8AEF7D00D735C1 /* SwiftyMarkdownExampleTests.swift */; }; F4CE98CC1C8AEF7D00D735C1 /* SwiftyMarkdownExampleUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4CE98CB1C8AEF7D00D735C1 /* SwiftyMarkdownExampleUITests.swift */; }; + F4EAB653244179FE00206782 /* example.md in Resources */ = {isa = PBXBuildFile; fileRef = F4576C2E2437F67B0013E2B6 /* example.md */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -62,8 +62,7 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ - F421DD951C8AF34F00B86D66 /* example.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = example.md; sourceTree = ""; }; - F4576C2E2437F67B0013E2B6 /* example copy.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = "example copy.md"; sourceTree = ""; }; + F4576C2E2437F67B0013E2B6 /* example.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = example.md; sourceTree = ""; }; F4B4A44923E4E17400550249 /* SwiftyMarkdownExample macOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "SwiftyMarkdownExample macOS.app"; sourceTree = BUILT_PRODUCTS_DIR; }; F4B4A44B23E4E17400550249 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; F4B4A44D23E4E17400550249 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; @@ -173,8 +172,7 @@ F4CE98B21C8AEF7D00D735C1 /* Assets.xcassets */, F4CE98B41C8AEF7D00D735C1 /* LaunchScreen.storyboard */, F4CE98B71C8AEF7D00D735C1 /* Info.plist */, - F4576C2E2437F67B0013E2B6 /* example copy.md */, - F421DD951C8AF34F00B86D66 /* example.md */, + F4576C2E2437F67B0013E2B6 /* example.md */, ); path = SwiftyMarkdownExample; sourceTree = ""; @@ -345,8 +343,8 @@ buildActionMask = 2147483647; files = ( F4CE98B61C8AEF7D00D735C1 /* LaunchScreen.storyboard in Resources */, + F4EAB653244179FE00206782 /* example.md in Resources */, F4CE98B31C8AEF7D00D735C1 /* Assets.xcassets in Resources */, - F421DD991C8AF4E900B86D66 /* example.md in Resources */, F4CE98B11C8AEF7D00D735C1 /* Main.storyboard in Resources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/Example/SwiftyMarkdownExample/example copy.md b/Example/SwiftyMarkdownExample/example copy.md deleted file mode 100644 index e7c927d..0000000 --- a/Example/SwiftyMarkdownExample/example copy.md +++ /dev/null @@ -1,50 +0,0 @@ -# Swifty Markdown - -SwiftyMarkdown is a Swift-based *Markdown* parser that converts *Markdown* files or strings into **NSAttributedStrings**. It uses sensible defaults and supports dynamic type, even with custom fonts. - -Show Images From Your App Bundle! ---- -![Image](bubble) - -Customise fonts and colors easily in a Swift-like way: - - md.code.fontName = "CourierNewPSMT" - - md.h2.fontName = "AvenirNextCondensed-Medium" - md.h2.color = UIColor.redColor() - md.h2.alignment = .center - -It supports the standard Markdown syntax, like *italics*, _underline italics_, **bold**, `backticks for code`, ~~strikethrough~~, and headings. - -It ignores random * and correctly handles escaped \*asterisks\* and \_underlines\_ and \`backticks\`. It also supports inline Markdown [Links](http://voyagetravelapps.com/). - -> It also now supports blockquotes -> and it supports whole-line italic and bold styles so you can go completely wild with styling! Wow! Such styles! Much fun! - -**Lists** - -- It Supports -- Unordered -- Lists - - Indented item with a longer string to make sure indentation is consistent - - Second level indent with a longer string to make sure indentation is consistent -- List item with a longer string to make sure indentation is consistent - -1. And -1. Ordered -1. Lists - 1. Indented item - 1. Second level indent -1. (Use `1.` as the list item identifier) -1. List item -1. List item - - Mix - - List styles -1. List item with a longer string to make sure indentation is consistent -1. List item -1. List item -1. List item -1. List item - - - diff --git a/Example/SwiftyMarkdownExample/example.md b/Example/SwiftyMarkdownExample/example.md index 17e1ce3..e7c927d 100644 --- a/Example/SwiftyMarkdownExample/example.md +++ b/Example/SwiftyMarkdownExample/example.md @@ -1 +1,50 @@ -[a](b) +# Swifty Markdown + +SwiftyMarkdown is a Swift-based *Markdown* parser that converts *Markdown* files or strings into **NSAttributedStrings**. It uses sensible defaults and supports dynamic type, even with custom fonts. + +Show Images From Your App Bundle! +--- +![Image](bubble) + +Customise fonts and colors easily in a Swift-like way: + + md.code.fontName = "CourierNewPSMT" + + md.h2.fontName = "AvenirNextCondensed-Medium" + md.h2.color = UIColor.redColor() + md.h2.alignment = .center + +It supports the standard Markdown syntax, like *italics*, _underline italics_, **bold**, `backticks for code`, ~~strikethrough~~, and headings. + +It ignores random * and correctly handles escaped \*asterisks\* and \_underlines\_ and \`backticks\`. It also supports inline Markdown [Links](http://voyagetravelapps.com/). + +> It also now supports blockquotes +> and it supports whole-line italic and bold styles so you can go completely wild with styling! Wow! Such styles! Much fun! + +**Lists** + +- It Supports +- Unordered +- Lists + - Indented item with a longer string to make sure indentation is consistent + - Second level indent with a longer string to make sure indentation is consistent +- List item with a longer string to make sure indentation is consistent + +1. And +1. Ordered +1. Lists + 1. Indented item + 1. Second level indent +1. (Use `1.` as the list item identifier) +1. List item +1. List item + - Mix + - List styles +1. List item with a longer string to make sure indentation is consistent +1. List item +1. List item +1. List item +1. List item + + + diff --git a/Sources/SwiftyMarkdown/SwiftyMarkdown.swift b/Sources/SwiftyMarkdown/SwiftyMarkdown.swift index 6749f29..73c8d38 100644 --- a/Sources/SwiftyMarkdown/SwiftyMarkdown.swift +++ b/Sources/SwiftyMarkdown/SwiftyMarkdown.swift @@ -188,7 +188,7 @@ If that is not set, then the system default will be used. CharacterRuleTag(tag: "(", type: .metadataOpen), CharacterRuleTag(tag: ")", type: .metadataClose) ], styles: [1 : CharacterStyle.link], metadataLookup: false, definesBoundary: true), - CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.code], shouldCancelRemainingTags: true, balancedTags: true), + CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.code], shouldCancelRemainingRules: true, balancedTags: true), CharacterRule(primaryTag:CharacterRuleTag(tag: "~", type: .repeating), otherTags : [], styles: [2 : CharacterStyle.strikethrough], minTags:2 , maxTags:2), CharacterRule(primaryTag: CharacterRuleTag(tag: "*", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2), CharacterRule(primaryTag: CharacterRuleTag(tag: "_", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2) diff --git a/Sources/SwiftyMarkdown/SwiftyScanner.swift b/Sources/SwiftyMarkdown/SwiftyScanner.swift index 85947b2..1834e6a 100644 --- a/Sources/SwiftyMarkdown/SwiftyScanner.swift +++ b/Sources/SwiftyMarkdown/SwiftyScanner.swift @@ -1,3 +1,10 @@ +// +// SwiftyScanner.swift +// +// +// Created by Simon Fairbairn on 04/04/2020. +// + // // SwiftyScanner.swift // SwiftyMarkdown @@ -10,50 +17,548 @@ import os.log extension OSLog { private static var subsystem = "SwiftyScanner" - static let swiftyScannerTokenising = OSLog(subsystem: subsystem, category: "Swifty Scanner Tokenising") - static let swiftyScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Peformance") + static let swiftyScanner = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner") + static let swiftyScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner Peformance") } -/// Swifty Scanning Protocol -public protocol SwiftyScanning { - var metadataLookup : [String : String] { get set } - func scan( _ string : String, with rule : CharacterRule) -> [Token] - func scan( _ tokens : [Token], with rule : CharacterRule) -> [Token] -} - -enum TagState { - case none +enum RepeatingTagType { case open - case intermediate - case closed + case either + case close + case neither } -class SwiftyScanner : SwiftyScanning { - var metadataLookup: [String : String] = [:] +struct TagGroup { + let groupID = UUID().uuidString + var tagRanges : [ClosedRange] + var tagType : RepeatingTagType = .open + var count = 1 +} + +class SwiftyScanner { + var elements : [Element] + let rule : CharacterRule + let metadata : [String : String] + var pointer : Int = 0 - init() { + var spaceAndNewLine = CharacterSet.whitespacesAndNewlines + var tagGroups : [TagGroup] = [] + + var isMetadataOpen = false + + + var enableLog = (ProcessInfo.processInfo.environment["SwiftyScannerScanner"] != nil) + + let currentPerfomanceLog = PerformanceLog(with: "SwiftyScannerScannerPerformanceLogging", identifier: "Scanner", log: OSLog.swiftyScannerPerformance) + let log = PerformanceLog(with: "SwiftyScannerScanner", identifier: "Scanner", log: OSLog.swiftyScanner) + + + enum Position { + case forward(Int) + case backward(Int) } - func scan(_ string: String, with rule: CharacterRule) -> [Token] { - return [] + init( withElements elements : [Element], rule : CharacterRule, metadata : [String : String]) { + self.elements = elements + self.rule = rule + self.currentPerfomanceLog.start() + self.metadata = metadata } - func scan(_ tokens: [Token], with rule: CharacterRule) -> [Token] { - return tokens + func elementsBetweenCurrentPosition( and newPosition : Position ) -> [Element]? { + + let newIdx : Int + var isForward = true + switch newPosition { + case .backward(let positions): + isForward = false + newIdx = pointer - positions + if newIdx < 0 { + return nil + } + case .forward(let positions): + newIdx = pointer + positions + if newIdx >= self.elements.count { + return nil + } + } + + + let range : ClosedRange = ( isForward ) ? self.pointer...newIdx : newIdx...self.pointer + return Array(self.elements[range]) } + + + func element( for position : Position ) -> Element? { + let newIdx : Int + switch position { + case .backward(let positions): + newIdx = pointer - positions + if newIdx < 0 { + return nil + } + case .forward(let positions): + newIdx = pointer + positions + if newIdx >= self.elements.count { + return nil + } + } + return self.elements[newIdx] + } + + + func positionIsEqualTo( character : Character, direction : Position ) -> Bool { + guard let validElement = self.element(for: direction) else { + return false + } + return validElement.character == character + } + + func positionContains( characters : [Character], direction : Position ) -> Bool { + guard let validElement = self.element(for: direction) else { + return false + } + return characters.contains(validElement.character) + } + + func isEscaped() -> Bool { + let isEscaped = self.positionContains(characters: self.rule.escapeCharacters, direction: .backward(1)) + if isEscaped { + self.elements[self.pointer - 1].type = .escape + } + return isEscaped + } + + func range( for tag : String? ) -> ClosedRange? { -} - -struct TokenGroup { - enum TokenGroupType { - case string - case tag - case escape + guard let tag = tag else { + return nil + } + + guard let openChar = tag.first else { + return nil + } + + if self.pointer == self.elements.count { + return nil + } + + if self.elements[self.pointer].character != openChar { + return nil + } + + if isEscaped() { + return nil + } + + let range : ClosedRange + if tag.count > 1 { + guard let elements = self.elementsBetweenCurrentPosition(and: .forward(tag.count - 1) ) else { + return nil + } + // If it's already a tag, then it should be ignored + if elements.filter({ $0.type != .string }).count > 0 { + return nil + } + if elements.map( { String($0.character) }).joined() != tag { + return nil + } + let endIdx = (self.pointer + tag.count - 1) + for i in self.pointer...endIdx { + self.elements[i].type = .tag + } + range = self.pointer...endIdx + self.pointer += tag.count + } else { + // If it's already a tag, then it should be ignored + if self.elements[self.pointer].type != .string { + return nil + } + self.elements[self.pointer].type = .tag + range = self.pointer...self.pointer + self.pointer += 1 + } + return range } - let string : String - let isEscaped : Bool - let type : TokenGroupType - var state : TagState = .none + + func resetTagGroup( withID id : String ) { + if let idx = self.tagGroups.firstIndex(where: { $0.groupID == id }) { + for range in self.tagGroups[idx].tagRanges { + self.resetTag(in: range) + } + self.tagGroups.remove(at: idx) + } + self.isMetadataOpen = false + } + + func resetTag( in range : ClosedRange) { + for idx in range { + self.elements[idx].type = .string + } + } + + func resetLastTag( for range : inout [ClosedRange]) { + guard let last = range.last else { + return + } + for idx in last { + self.elements[idx].type = .string + } + } + + func closeTag( _ tag : String, withGroupID id : String ) { + + guard let tagIdx = self.tagGroups.firstIndex(where: { $0.groupID == id }) else { + return + } + + var metadataString = "" + if self.isMetadataOpen { + let metadataCloseRange = self.tagGroups[tagIdx].tagRanges.removeLast() + let metadataOpenRange = self.tagGroups[tagIdx].tagRanges.removeLast() + + if metadataOpenRange.upperBound + 1 == (metadataCloseRange.lowerBound) { + if self.enableLog { + os_log("Nothing between the tags", log: OSLog.swiftyScanner, type:.info , self.rule.description) + } + } else { + for idx in (metadataOpenRange.upperBound)...(metadataCloseRange.lowerBound) { + self.elements[idx].type = .metadata + if self.rule.definesBoundary { + self.elements[idx].boundaryCount += 1 + } + } + + + let key = self.elements[metadataOpenRange.upperBound + 1.. 0 { + if remainingTags >= self.rule.maxTags { + remainingTags -= self.rule.maxTags + if let style = self.rule.styles[ self.rule.maxTags ] { + if !styles.contains(where: { $0.isEqualTo(style)}) { + styles.append(style) + } + } + } + if let style = self.rule.styles[remainingTags] { + remainingTags -= remainingTags + if !styles.contains(where: { $0.isEqualTo(style)}) { + styles.append(style) + } + } + } + + for idx in (openRange.upperBound)...(closeRange.lowerBound) { + self.elements[idx].styles.append(contentsOf: styles) + self.elements[idx].metadata.append(metadataString) + if self.rule.definesBoundary { + self.elements[idx].boundaryCount += 1 + } + if self.rule.shouldCancelRemainingRules { + self.elements[idx].boundaryCount = 1000 + } + } + + if self.rule.isRepeatingTag { + let difference = ( openRange.upperBound - openRange.lowerBound ) - (closeRange.upperBound - closeRange.lowerBound) + switch difference { + case 1...: + shouldRemove = false + self.tagGroups[tagIdx].count = difference + self.tagGroups[tagIdx].tagRanges.append( openRange.upperBound - (abs(difference) - 1)...openRange.upperBound ) + case ...(-1): + for idx in closeRange.upperBound - (abs(difference) - 1)...closeRange.upperBound { + self.elements[idx].type = .string + } + default: + break + } + } + + } + if shouldRemove { + self.tagGroups.removeAll(where: { $0.groupID == id }) + } + self.isMetadataOpen = false + } + + func emptyRanges( _ ranges : inout [ClosedRange] ) { + while !ranges.isEmpty { + self.resetLastTag(for: &ranges) + ranges.removeLast() + } + } + + func scanNonRepeatingTags() { + var groupID = "" + let closeTag = self.rule.tag(for: .close)?.tag + let metadataOpen = self.rule.tag(for: .metadataOpen)?.tag + let metadataClose = self.rule.tag(for: .metadataClose)?.tag + + while self.pointer < self.elements.count { + if self.enableLog { + os_log("CHARACTER: %@", log: OSLog.swiftyScanner, type:.info , String(self.elements[self.pointer].character)) + } + + if let range = self.range(for: metadataClose) { + if self.isMetadataOpen { + guard let groupIdx = self.tagGroups.firstIndex(where: { $0.groupID == groupID }) else { + self.pointer += 1 + continue + } + + guard !self.tagGroups.isEmpty else { + self.resetTagGroup(withID: groupID) + continue + } + + guard self.isMetadataOpen else { + + self.resetTagGroup(withID: groupID) + continue + } + if self.enableLog { + os_log("Closing metadata tag found. Closing tag with ID %@", log: OSLog.swiftyScanner, type:.info , groupID) + } + self.tagGroups[groupIdx].tagRanges.append(range) + self.closeTag(closeTag!, withGroupID: groupID) + self.isMetadataOpen = false + continue + } else { + self.resetTag(in: range) + self.pointer -= metadataClose!.count + } + + } + + if let openRange = self.range(for: self.rule.primaryTag.tag) { + if self.isMetadataOpen { + self.resetTagGroup(withID: groupID) + } + + let tagGroup = TagGroup(tagRanges: [openRange]) + groupID = tagGroup.groupID + if self.enableLog { + os_log("New open tag found. Starting new Group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID) + } + if self.rule.isRepeatingTag { + + } + + self.tagGroups.append(tagGroup) + continue + } + + if let range = self.range(for: closeTag) { + guard !self.tagGroups.isEmpty else { + if self.enableLog { + os_log("No open tags exist, resetting this close tag", log: OSLog.swiftyScanner, type:.info) + } + self.resetTag(in: range) + continue + } + self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range) + groupID = self.tagGroups[self.tagGroups.count - 1].groupID + if self.enableLog { + os_log("New close tag found. Appending to group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID) + } + guard metadataOpen != nil else { + if self.enableLog { + os_log("No metadata tags exist, closing valid tag with ID %@", log: OSLog.swiftyScanner, type:.info , groupID) + } + self.closeTag(closeTag!, withGroupID: groupID) + continue + } + + guard self.pointer != self.elements.count else { + continue + } + + guard let range = self.range(for: metadataOpen) else { + if self.enableLog { + os_log("No metadata tag found, resetting group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID) + } + self.resetTagGroup(withID: groupID) + continue + } + self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range) + self.isMetadataOpen = true + continue + } + + + if let range = self.range(for: metadataOpen) { + if self.enableLog { + os_log("Multiple open metadata tags found!", log: OSLog.swiftyScanner, type:.info , groupID) + } + self.resetTag(in: range) + self.resetTagGroup(withID: groupID) + self.isMetadataOpen = false + continue + } + self.pointer += 1 + } + } + + func scanRepeatingTags() { + + var groupID = "" + let escapeCharacters = "" //self.rule.escapeCharacters.map( { String( $0 ) }).joined() + let unionSet = spaceAndNewLine.union(CharacterSet(charactersIn: escapeCharacters)) + while self.pointer < self.elements.count { + if self.enableLog { + os_log("CHARACTER: %@", log: OSLog.swiftyScanner, type:.info , String(self.elements[self.pointer].character)) + } + + if var openRange = self.range(for: self.rule.primaryTag.tag) { + + if self.elements[openRange].first?.boundaryCount == 1000 { + self.resetTag(in: openRange) + continue + } + + var count = 1 + var tagType : RepeatingTagType = .open + if let prevElement = self.element(for: .backward(self.rule.primaryTag.tag.count + 1)) { + if !unionSet.containsUnicodeScalars(of: prevElement.character) { + tagType = .either + } + } else { + tagType = .open + } + + while let nextRange = self.range(for: self.rule.primaryTag.tag) { + count += 1 + openRange = openRange.lowerBound...nextRange.upperBound + } + + if self.rule.minTags > 1 { + if (openRange.upperBound - openRange.lowerBound) + 1 < self.rule.minTags { + self.resetTag(in: openRange) + os_log("Tag does not meet minimum length", log: .swiftyScanner, type: .info) + continue + } + } + + var validTagGroup = true + if let nextElement = self.element(for: .forward(0)) { + if unionSet.containsUnicodeScalars(of: nextElement.character) { + if tagType == .either { + tagType = .close + } else { + validTagGroup = tagType != .open + } + } + } else { + if tagType == .either { + tagType = .close + } else { + validTagGroup = tagType != .open + } + } + + if !validTagGroup { + if self.enableLog { + os_log("Tag has whitespace on both sides", log: .swiftyScanner, type: .info) + } + self.resetTag(in: openRange) + continue + } + + if let idx = tagGroups.firstIndex(where: { $0.groupID == groupID }) { + if tagType == .either { + if tagGroups[idx].count == count { + self.tagGroups[idx].tagRanges.append(openRange) + self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID) + + if let last = self.tagGroups.last { + groupID = last.groupID + } + + continue + } + } else { + if let prevRange = tagGroups[idx].tagRanges.first { + if self.elements[prevRange].first?.boundaryCount == self.elements[openRange].first?.boundaryCount { + self.tagGroups[idx].tagRanges.append(openRange) + self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID) + } + } + continue + } + } + var tagGroup = TagGroup(tagRanges: [openRange]) + groupID = tagGroup.groupID + tagGroup.tagType = tagType + tagGroup.count = count + + if self.enableLog { + os_log("New open tag found with characters %@. Starting new Group with ID %@", log: OSLog.swiftyScanner, type:.info, self.elements[openRange].map( { String($0.character) }).joined(), groupID) + } + + self.tagGroups.append(tagGroup) + continue + } + + self.pointer += 1 + } + } + + + func scan() -> [Element] { + + guard self.elements.filter({ $0.type == .string }).map({ String($0.character) }).joined().contains(self.rule.primaryTag.tag) else { + return self.elements + } + + self.currentPerfomanceLog.tag(with: "Beginning \(self.rule.primaryTag.tag)") + + if self.enableLog { + os_log("RULE: %@", log: OSLog.swiftyScanner, type:.info , self.rule.description) + } + + if self.rule.isRepeatingTag { + self.scanRepeatingTags() + } else { + self.scanNonRepeatingTags() + } + + for tagGroup in self.tagGroups { + self.resetTagGroup(withID: tagGroup.groupID) + } + + if self.enableLog { + for element in self.elements { + print(element) + } + } + return self.elements + } } diff --git a/Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift b/Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift deleted file mode 100644 index 8d8170a..0000000 --- a/Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift +++ /dev/null @@ -1,565 +0,0 @@ -// -// File.swift -// -// -// Created by Simon Fairbairn on 04/04/2020. -// - -// -// SwiftyScanner.swift -// SwiftyMarkdown -// -// Created by Simon Fairbairn on 04/02/2020. -// - -import Foundation -import os.log - - -extension OSLog { - private static var subsystem = "SwiftyScanner" - static let swiftyScannerScanner = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner") - static let swiftyScannerScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner Peformance") -} - -enum RepeatingTagType { - case open - case either - case close - case neither -} - -struct TagGroup { - let groupID = UUID().uuidString - var tagRanges : [ClosedRange] - var tagType : RepeatingTagType = .open - var count = 1 -} - -class SwiftyScannerNonRepeating { - var elements : [Element] - let rule : CharacterRule - let metadata : [String : String] - var pointer : Int = 0 - - var spaceAndNewLine = CharacterSet.whitespacesAndNewlines - var tagGroups : [TagGroup] = [] - - var isMetadataOpen = false - - - var enableLog = (ProcessInfo.processInfo.environment["SwiftyScannerScanner"] != nil) - - let currentPerfomanceLog = PerformanceLog(with: "SwiftyScannerScannerPerformanceLogging", identifier: "Scanner", log: OSLog.swiftyScannerPerformance) - let log = PerformanceLog(with: "SwiftyScannerScanner", identifier: "Scanner", log: OSLog.swiftyScannerScanner) - - - - enum Position { - case forward(Int) - case backward(Int) - } - - init( withElements elements : [Element], rule : CharacterRule, metadata : [String : String]) { - self.elements = elements - self.rule = rule - self.currentPerfomanceLog.start() - self.metadata = metadata - } - - func elementsBetweenCurrentPosition( and newPosition : Position ) -> [Element]? { - - let newIdx : Int - var isForward = true - switch newPosition { - case .backward(let positions): - isForward = false - newIdx = pointer - positions - if newIdx < 0 { - return nil - } - case .forward(let positions): - newIdx = pointer + positions - if newIdx >= self.elements.count { - return nil - } - } - - - let range : ClosedRange = ( isForward ) ? self.pointer...newIdx : newIdx...self.pointer - return Array(self.elements[range]) - } - - - func element( for position : Position ) -> Element? { - let newIdx : Int - switch position { - case .backward(let positions): - newIdx = pointer - positions - if newIdx < 0 { - return nil - } - case .forward(let positions): - newIdx = pointer + positions - if newIdx >= self.elements.count { - return nil - } - } - return self.elements[newIdx] - } - - - func positionIsEqualTo( character : Character, direction : Position ) -> Bool { - guard let validElement = self.element(for: direction) else { - return false - } - return validElement.character == character - } - - func positionContains( characters : [Character], direction : Position ) -> Bool { - guard let validElement = self.element(for: direction) else { - return false - } - return characters.contains(validElement.character) - } - - func isEscaped() -> Bool { - let isEscaped = self.positionContains(characters: self.rule.escapeCharacters, direction: .backward(1)) - if isEscaped { - self.elements[self.pointer - 1].type = .escape - } - return isEscaped - } - - func range( for tag : String? ) -> ClosedRange? { - - guard let tag = tag else { - return nil - } - - guard let openChar = tag.first else { - return nil - } - - if self.pointer == self.elements.count { - return nil - } - - if self.elements[self.pointer].character != openChar { - return nil - } - - if isEscaped() { - return nil - } - - let range : ClosedRange - if tag.count > 1 { - guard let elements = self.elementsBetweenCurrentPosition(and: .forward(tag.count - 1) ) else { - return nil - } - // If it's already a tag, then it should be ignored - if elements.filter({ $0.type != .string }).count > 0 { - return nil - } - if elements.map( { String($0.character) }).joined() != tag { - return nil - } - let endIdx = (self.pointer + tag.count - 1) - for i in self.pointer...endIdx { - self.elements[i].type = .tag - } - range = self.pointer...endIdx - self.pointer += tag.count - } else { - // If it's already a tag, then it should be ignored - if self.elements[self.pointer].type != .string { - return nil - } - self.elements[self.pointer].type = .tag - range = self.pointer...self.pointer - self.pointer += 1 - } - return range - } - - - func resetTagGroup( withID id : String ) { - if let idx = self.tagGroups.firstIndex(where: { $0.groupID == id }) { - for range in self.tagGroups[idx].tagRanges { - self.resetTag(in: range) - } - self.tagGroups.remove(at: idx) - } - self.isMetadataOpen = false - } - - func resetTag( in range : ClosedRange) { - for idx in range { - self.elements[idx].type = .string - } - } - - func resetLastTag( for range : inout [ClosedRange]) { - guard let last = range.last else { - return - } - for idx in last { - self.elements[idx].type = .string - } - } - - func closeTag( _ tag : String, withGroupID id : String ) { - - guard let tagIdx = self.tagGroups.firstIndex(where: { $0.groupID == id }) else { - return - } - - var metadataString = "" - if self.isMetadataOpen { - let metadataCloseRange = self.tagGroups[tagIdx].tagRanges.removeLast() - let metadataOpenRange = self.tagGroups[tagIdx].tagRanges.removeLast() - - if metadataOpenRange.upperBound + 1 == (metadataCloseRange.lowerBound) { - if self.enableLog { - os_log("Nothing between the tags", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description) - } - } else { - for idx in (metadataOpenRange.upperBound)...(metadataCloseRange.lowerBound) { - self.elements[idx].type = .metadata - if self.rule.definesBoundary { - self.elements[idx].boundaryCount += 1 - } - } - - - let key = self.elements[metadataOpenRange.upperBound + 1.. 0 { - if remainingTags >= self.rule.maxTags { - remainingTags -= self.rule.maxTags - if let style = self.rule.styles[ self.rule.maxTags ] { - if !styles.contains(where: { $0.isEqualTo(style)}) { - styles.append(style) - } - } - } - if let style = self.rule.styles[remainingTags] { - remainingTags -= remainingTags - if !styles.contains(where: { $0.isEqualTo(style)}) { - styles.append(style) - } - } - } - - for idx in (openRange.upperBound)...(closeRange.lowerBound) { - self.elements[idx].styles.append(contentsOf: styles) - self.elements[idx].metadata.append(metadataString) - if self.rule.definesBoundary { - self.elements[idx].boundaryCount += 1 - } - if self.rule.shouldCancelRemainingRules { - self.elements[idx].boundaryCount = 1000 - } - } - - if self.rule.isRepeatingTag { - let difference = ( openRange.upperBound - openRange.lowerBound ) - (closeRange.upperBound - closeRange.lowerBound) - switch difference { - case 1...: - shouldRemove = false - self.tagGroups[tagIdx].count = difference - self.tagGroups[tagIdx].tagRanges.append( openRange.upperBound - (abs(difference) - 1)...openRange.upperBound ) - case ...(-1): - for idx in closeRange.upperBound - (abs(difference) - 1)...closeRange.upperBound { - self.elements[idx].type = .string - } - default: - break - } - } - - } - if shouldRemove { - self.tagGroups.removeAll(where: { $0.groupID == id }) - } - self.isMetadataOpen = false - } - - func emptyRanges( _ ranges : inout [ClosedRange] ) { - while !ranges.isEmpty { - self.resetLastTag(for: &ranges) - ranges.removeLast() - } - } - - func scanNonRepeatingTags() { - var groupID = "" - let closeTag = self.rule.tag(for: .close)?.tag - let metadataOpen = self.rule.tag(for: .metadataOpen)?.tag - let metadataClose = self.rule.tag(for: .metadataClose)?.tag - - while self.pointer < self.elements.count { - if self.enableLog { - os_log("CHARACTER: %@", log: OSLog.swiftyScannerScanner, type:.info , String(self.elements[self.pointer].character)) - } - - if let range = self.range(for: metadataClose) { - if self.isMetadataOpen { - guard let groupIdx = self.tagGroups.firstIndex(where: { $0.groupID == groupID }) else { - self.pointer += 1 - continue - } - - guard !self.tagGroups.isEmpty else { - self.resetTagGroup(withID: groupID) - continue - } - - guard self.isMetadataOpen else { - - self.resetTagGroup(withID: groupID) - continue - } - if self.enableLog { - os_log("Closing metadata tag found. Closing tag with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID) - } - self.tagGroups[groupIdx].tagRanges.append(range) - self.closeTag(closeTag!, withGroupID: groupID) - self.isMetadataOpen = false - continue - } else { - self.resetTag(in: range) - self.pointer -= metadataClose!.count - } - - } - - if let openRange = self.range(for: self.rule.primaryTag.tag) { - if self.isMetadataOpen { - self.resetTagGroup(withID: groupID) - } - - let tagGroup = TagGroup(tagRanges: [openRange]) - groupID = tagGroup.groupID - if self.enableLog { - os_log("New open tag found. Starting new Group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID) - } - if self.rule.isRepeatingTag { - - } - - self.tagGroups.append(tagGroup) - continue - } - - if let range = self.range(for: closeTag) { - guard !self.tagGroups.isEmpty else { - if self.enableLog { - os_log("No open tags exist, resetting this close tag", log: OSLog.swiftyScannerScanner, type:.info) - } - self.resetTag(in: range) - continue - } - self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range) - groupID = self.tagGroups[self.tagGroups.count - 1].groupID - if self.enableLog { - os_log("New close tag found. Appending to group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID) - } - guard metadataOpen != nil else { - if self.enableLog { - os_log("No metadata tags exist, closing valid tag with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID) - } - self.closeTag(closeTag!, withGroupID: groupID) - continue - } - - guard self.pointer != self.elements.count else { - continue - } - - guard let range = self.range(for: metadataOpen) else { - if self.enableLog { - os_log("No metadata tag found, resetting group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID) - } - self.resetTagGroup(withID: groupID) - continue - } - self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range) - self.isMetadataOpen = true - continue - } - - - if let range = self.range(for: metadataOpen) { - if self.enableLog { - os_log("Multiple open metadata tags found!", log: OSLog.swiftyScannerScanner, type:.info , groupID) - } - self.resetTag(in: range) - self.resetTagGroup(withID: groupID) - self.isMetadataOpen = false - continue - } - self.pointer += 1 - } - } - - func scanRepeatingTags() { - - var groupID = "" - let escapeCharacters = "" //self.rule.escapeCharacters.map( { String( $0 ) }).joined() - let unionSet = spaceAndNewLine.union(CharacterSet(charactersIn: escapeCharacters)) - while self.pointer < self.elements.count { - if self.enableLog { - os_log("CHARACTER: %@", log: OSLog.swiftyScannerScanner, type:.info , String(self.elements[self.pointer].character)) - } - - if var openRange = self.range(for: self.rule.primaryTag.tag) { - - if self.elements[openRange].first?.boundaryCount == 1000 { - self.resetTag(in: openRange) - continue - } - - var count = 1 - var tagType : RepeatingTagType = .open - if let prevElement = self.element(for: .backward(self.rule.primaryTag.tag.count + 1)) { - if !unionSet.containsUnicodeScalars(of: prevElement.character) { - tagType = .either - } - } else { - tagType = .open - } - - while let nextRange = self.range(for: self.rule.primaryTag.tag) { - count += 1 - openRange = openRange.lowerBound...nextRange.upperBound - } - - if self.rule.minTags > 1 { - if (openRange.upperBound - openRange.lowerBound) + 1 < self.rule.minTags { - self.resetTag(in: openRange) - os_log("Tag does not meet minimum length", log: .swiftyScannerScanner, type: .info) - continue - } - } - - var validTagGroup = true - if let nextElement = self.element(for: .forward(0)) { - if unionSet.containsUnicodeScalars(of: nextElement.character) { - if tagType == .either { - tagType = .close - } else { - validTagGroup = tagType != .open - } - } - } else { - if tagType == .either { - tagType = .close - } else { - validTagGroup = tagType != .open - } - } - - if !validTagGroup { - if self.enableLog { - os_log("Tag has whitespace on both sides", log: .swiftyScannerScanner, type: .info) - } - self.resetTag(in: openRange) - continue - } - - if let idx = tagGroups.firstIndex(where: { $0.groupID == groupID }) { - if tagType == .either { - if tagGroups[idx].count == count { - self.tagGroups[idx].tagRanges.append(openRange) - self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID) - - if let last = self.tagGroups.last { - groupID = last.groupID - } - - continue - } - } else { - if let prevRange = tagGroups[idx].tagRanges.first { - if self.elements[prevRange].first?.boundaryCount == self.elements[openRange].first?.boundaryCount { - self.tagGroups[idx].tagRanges.append(openRange) - self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID) - } - } - continue - } - } - var tagGroup = TagGroup(tagRanges: [openRange]) - groupID = tagGroup.groupID - tagGroup.tagType = tagType - tagGroup.count = count - - if self.enableLog { - os_log("New open tag found with characters %@. Starting new Group with ID %@", log: OSLog.swiftyScannerScanner, type:.info, self.elements[openRange].map( { String($0.character) }).joined(), groupID) - } - - self.tagGroups.append(tagGroup) - continue - } - - self.pointer += 1 - } - } - - - func scan() -> [Element] { - - guard self.elements.filter({ $0.type == .string }).map({ String($0.character) }).joined().contains(self.rule.primaryTag.tag) else { - return self.elements - } - - self.currentPerfomanceLog.tag(with: "Beginning \(self.rule.primaryTag.tag)") - - if self.enableLog { - os_log("RULE: %@", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description) - } - - if self.rule.isRepeatingTag { - self.scanRepeatingTags() - } else { - self.scanNonRepeatingTags() - } - - for tagGroup in self.tagGroups { - self.resetTagGroup(withID: tagGroup.groupID) - } - - if self.enableLog { - for element in self.elements { - print(element) - } - } - return self.elements - } -} diff --git a/Sources/SwiftyMarkdown/SwiftyTokeniser.swift b/Sources/SwiftyMarkdown/SwiftyTokeniser.swift index 05c6653..cce1a97 100644 --- a/Sources/SwiftyMarkdown/SwiftyTokeniser.swift +++ b/Sources/SwiftyMarkdown/SwiftyTokeniser.swift @@ -23,7 +23,6 @@ public class SwiftyTokeniser { let totalPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Total Run Time", log: OSLog.performance) let currentPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Current", log: OSLog.performance) - var scanner : SwiftyScanning! public var metadataLookup : [String : String] = [:] let newlines = CharacterSet.newlines @@ -53,7 +52,7 @@ public class SwiftyTokeniser { /// /// - Parameter inputString: A string to have the CharacterRules in `self.rules` applied to public func process( _ inputString : String ) -> [Token] { - var currentTokens = [Token(type: .string, inputString: inputString)] + let currentTokens = [Token(type: .string, inputString: inputString)] guard rules.count > 0 else { return currentTokens } @@ -83,19 +82,13 @@ public class SwiftyTokeniser { while !mutableRules.isEmpty { let nextRule = mutableRules.removeFirst() - - if nextRule.isRepeatingTag { - self.scanner = SwiftyScanner() - self.scanner.metadataLookup = self.metadataLookup - } - if enableLog { os_log("------------------------------", log: .tokenising, type: .info) os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description) } self.currentPerfomanceLog.tag(with: "(start rule %@)") - let scanner = SwiftyScannerNonRepeating(withElements: elementArray, rule: nextRule, metadata: self.metadataLookup) + let scanner = SwiftyScanner(withElements: elementArray, rule: nextRule, metadata: self.metadataLookup) elementArray = scanner.scan() } @@ -113,7 +106,6 @@ public class SwiftyTokeniser { tokens.append(token) } - var lastElement = elementArray.first! var accumulatedString = "" for element in elementArray { @@ -141,372 +133,6 @@ public class SwiftyTokeniser { } return output } - - -// -// -// /// In order to reinsert the original replacements into the new string token, the replacements -// /// need to be searched for in the incoming string one by one. -// /// -// /// Using the `newToken(fromSubstring:isReplacement:)` function ensures that any metadata and character styles -// /// are passed over into the newly created tokens. -// /// -// /// E.g. A string token that has an `outputString` of "This string AAAAA-BBBBB-CCCCC replacements", with -// /// a characterStyle of `bold` for the entire string, needs to be separated into the following tokens: -// /// -// /// - `string`: "This string " -// /// - `replacement`: "AAAAA-BBBBB-CCCCC" -// /// - `string`: " replacements" -// /// -// /// Each of these need to have a character style of `bold`. -// /// -// /// - Parameters: -// /// - replacements: An array of `replacement` tokens -// /// - token: The new `string` token that may contain replacement IDs contained in the `replacements` array -// func reinsertReplacements(_ replacements : [Token], from stringToken : Token ) -> [Token] { -// guard !stringToken.outputString.isEmpty && !replacements.isEmpty else { -// return [stringToken] -// } -// var outputTokens : [Token] = [] -// let scanner = Scanner(string: stringToken.outputString) -// scanner.charactersToBeSkipped = nil -// -// // Remove any replacements that don't appear in the incoming string -// var repTokens = replacements.filter({ stringToken.outputString.contains($0.inputString) }) -// -// var testString = "\n" -// while !scanner.isAtEnd { -// var outputString : String = "" -// if repTokens.count > 0 { -// testString = repTokens.removeFirst().inputString -// } -// -// if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) { -// if let nextString = scanner.scanUpToString(testString) { -// outputString = nextString -// outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false)) -// if let outputToken = scanner.scanString(testString) { -// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true)) -// } -// } else if let outputToken = scanner.scanString(testString) { -// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true)) -// } -// } else { -// var oldString : NSString? = nil -// var tokenString : NSString? = nil -// scanner.scanUpTo(testString, into: &oldString) -// if let nextString = oldString { -// outputString = nextString as String -// outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false)) -// scanner.scanString(testString, into: &tokenString) -// if let outputToken = tokenString as String? { -// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true)) -// } -// } else { -// scanner.scanString(testString, into: &tokenString) -// if let outputToken = tokenString as String? { -// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true)) -// } -// } -// } -// } -// return outputTokens -// } -// -// -// /// This function is necessary because a previously tokenised string might have -// /// -// /// Consider a previously tokenised string, where AAAAA-BBBBB-CCCCC represents a replaced \[link\](url) instance. -// /// -// /// The incoming tokens will look like this: -// /// -// /// - `string`: "A \*\*Bold" -// /// - `replacement` : "AAAAA-BBBBB-CCCCC" -// /// - `string`: " with a trailing string**" -// /// -// /// However, because the scanner can only tokenise individual strings, passing in the string values -// /// of these tokens individually and applying the styles will not correctly detect the starting and -// /// ending `repeatingTag` instances. (e.g. the scanner will see "A \*\*Bold", and then "AAAAA-BBBBB-CCCCC", -// /// and finally " with a trailing string\*\*") -// /// -// /// The strings need to be combined, so that they form a single string: -// /// A \*\*Bold AAAAA-BBBBB-CCCCC with a trailing string\*\*. -// /// This string is then parsed and tokenised so that it looks like this: -// /// -// /// - `string`: "A " -// /// - `repeatingTag`: "\*\*" -// /// - `string`: "Bold AAAAA-BBBBB-CCCCC with a trailing string" -// /// - `repeatingTag`: "\*\*" -// /// -// /// Finally, the replacements from the original incoming token array are searched for and pulled out -// /// of this new string, so the final result looks like this: -// /// -// /// - `string`: "A " -// /// - `repeatingTag`: "\*\*" -// /// - `string`: "Bold " -// /// - `replacement`: "AAAAA-BBBBB-CCCCC" -// /// - `string`: " with a trailing string" -// /// - `repeatingTag`: "\*\*" -// /// -// /// - Parameters: -// /// - tokens: The tokens to be combined, scanned, re-tokenised, and merged -// /// - rule: The character rule currently being applied -// func scanReplacementTokens( _ tokens : [Token], with rule : CharacterRule ) -> [Token] { -// guard tokens.count > 0 else { -// return [] -// } -// -// let combinedString = tokens.map({ $0.outputString }).joined() -// -// let nextTokens = self.scanner.scan(combinedString, with: rule) -// var replacedTokens = self.applyStyles(to: nextTokens, usingRule: rule) -// -// /// It's necessary here to check to see if the first token (which will always represent the styles -// /// to be applied from previous scans) has any existing metadata or character styles and apply them -// /// to *all* the string and replacement tokens found by the new scan. -// for idx in 0.. [Token] { -// -// // Only combine string and replacements that are next to each other. -// var newTokenSet : [Token] = [] -// var currentTokenSet : [Token] = [] -// for i in 0.. 0 else { -// return -// } -// -// let startIdx = index -// var endIdx : Int? = nil -// -// let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count -// // Try to find exact match first -// if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) { -// endIdx = nextTokenIdx -// } -// -// if endIdx == nil, let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count >= 1 && $0.id != theToken.id && !$0.isProcessed }) { -// endIdx = nextTokenIdx -// } -// guard let existentEnd = endIdx else { -// return -// } -// -// -// let styles : [CharacterStyling] = rule.styles[maxCount] ?? [] -// for i in startIdx.. rule.maxTags) ? rule.maxTags : tokens[existentEnd].count -// tokens[index].count = theToken.count - maxEnd -// tokens[existentEnd].count = tokens[existentEnd].count - maxEnd -// if maxEnd < rule.maxTags { -// self.handleClosingTagFromRepeatingTag(withIndex: index, in: &tokens, following: rule) -// } else { -// tokens[existentEnd].isProcessed = true -// tokens[index].isProcessed = true -// } -// -// -// } -// -// func applyStyles( to tokens : [Token], usingRule rule : CharacterRule ) -> [Token] { -// var mutableTokens : [Token] = tokens -// -// if enableLog { -// os_log("Applying styles to tokens: %@", log: .tokenising, type: .info, tokens.oslogDisplay ) -// } -// for idx in 0..