dnrops
/
SwiftyMarkdown
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleans up codebase and class names

This commit is contained in:
Simon Fairbairn 2020-04-11 16:08:02 +12:00
parent bcbf3bd38c
commit 61490b1e13
7 changed files with 593 additions and 1030 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Example/SwiftyMarkdownExample.xcodeproj/project.pbxproj
View File

@ -7,7 +7,6 @@
objects = {
/* Begin PBXBuildFile section */
F421DD991C8AF4E900B86D66 /* example.md in Resources */ = {isa = PBXBuildFile; fileRef = F421DD951C8AF34F00B86D66 /* example.md */; };
F4B4A44C23E4E17400550249 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4B4A44B23E4E17400550249 /* AppDelegate.swift */; };
F4B4A44E23E4E17400550249 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4B4A44D23E4E17400550249 /* ViewController.swift */; };
F4B4A45023E4E17400550249 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = F4B4A44F23E4E17400550249 /* Assets.xcassets */; };
@ -20,6 +19,7 @@
F4CE98B61C8AEF7D00D735C1 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = F4CE98B41C8AEF7D00D735C1 /* LaunchScreen.storyboard */; };
F4CE98C11C8AEF7D00D735C1 /* SwiftyMarkdownExampleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4CE98C01C8AEF7D00D735C1 /* SwiftyMarkdownExampleTests.swift */; };
F4CE98CC1C8AEF7D00D735C1 /* SwiftyMarkdownExampleUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4CE98CB1C8AEF7D00D735C1 /* SwiftyMarkdownExampleUITests.swift */; };
F4EAB653244179FE00206782 /* example.md in Resources */ = {isa = PBXBuildFile; fileRef = F4576C2E2437F67B0013E2B6 /* example.md */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@ -62,8 +62,7 @@
/* End PBXCopyFilesBuildPhase section */
/* Begin PBXFileReference section */
F421DD951C8AF34F00B86D66 /* example.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = example.md; sourceTree = "<group>"; };
F4576C2E2437F67B0013E2B6 /* example copy.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = "example copy.md"; sourceTree = "<group>"; };
F4576C2E2437F67B0013E2B6 /* example.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = example.md; sourceTree = "<group>"; };
F4B4A44923E4E17400550249 /* SwiftyMarkdownExample macOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "SwiftyMarkdownExample macOS.app"; sourceTree = BUILT_PRODUCTS_DIR; };
F4B4A44B23E4E17400550249 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
F4B4A44D23E4E17400550249 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
@ -173,8 +172,7 @@
F4CE98B21C8AEF7D00D735C1 /* Assets.xcassets */,
F4CE98B41C8AEF7D00D735C1 /* LaunchScreen.storyboard */,
F4CE98B71C8AEF7D00D735C1 /* Info.plist */,
F4576C2E2437F67B0013E2B6 /* example copy.md */,
F421DD951C8AF34F00B86D66 /* example.md */,
F4576C2E2437F67B0013E2B6 /* example.md */,
);
path = SwiftyMarkdownExample;
sourceTree = "<group>";
@ -345,8 +343,8 @@
buildActionMask = 2147483647;
files = (
F4CE98B61C8AEF7D00D735C1 /* LaunchScreen.storyboard in Resources */,
F4EAB653244179FE00206782 /* example.md in Resources */,
F4CE98B31C8AEF7D00D735C1 /* Assets.xcassets in Resources */,
F421DD991C8AF4E900B86D66 /* example.md in Resources */,
F4CE98B11C8AEF7D00D735C1 /* Main.storyboard in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;

50
Example/SwiftyMarkdownExample/example copy.md
View File

@ -1,50 +0,0 @@
# Swifty Markdown
SwiftyMarkdown is a Swift-based *Markdown* parser that converts *Markdown* files or strings into **NSAttributedStrings**. It uses sensible defaults and supports dynamic type, even with custom fonts.
Show Images From Your App Bundle!
---
![Image](bubble)
Customise fonts and colors easily in a Swift-like way:
md.code.fontName = "CourierNewPSMT"
md.h2.fontName = "AvenirNextCondensed-Medium"
md.h2.color = UIColor.redColor()
md.h2.alignment = .center
It supports the standard Markdown syntax, like *italics*, _underline italics_, **bold**, `backticks for code`, ~~strikethrough~~, and headings.
It ignores random * and correctly handles escaped \*asterisks\* and \_underlines\_ and \`backticks\`. It also supports inline Markdown [Links](http://voyagetravelapps.com/).
> It also now supports blockquotes
> and it supports whole-line italic and bold styles so you can go completely wild with styling! Wow! Such styles! Much fun!
**Lists**
- It Supports
- Unordered
- Lists
- Indented item with a longer string to make sure indentation is consistent
- Second level indent with a longer string to make sure indentation is consistent
- List item with a longer string to make sure indentation is consistent
1. And
1. Ordered
1. Lists
1. Indented item
1. Second level indent
1. (Use `1.` as the list item identifier)
1. List item
1. List item
- Mix
- List styles
1. List item with a longer string to make sure indentation is consistent
1. List item
1. List item
1. List item
1. List item

51
Example/SwiftyMarkdownExample/example.md
View File

@ -1 +1,50 @@
[a](b)
# Swifty Markdown
SwiftyMarkdown is a Swift-based *Markdown* parser that converts *Markdown* files or strings into **NSAttributedStrings**. It uses sensible defaults and supports dynamic type, even with custom fonts.
Show Images From Your App Bundle!
---
![Image](bubble)
Customise fonts and colors easily in a Swift-like way:
md.code.fontName = "CourierNewPSMT"
md.h2.fontName = "AvenirNextCondensed-Medium"
md.h2.color = UIColor.redColor()
md.h2.alignment = .center
It supports the standard Markdown syntax, like *italics*, _underline italics_, **bold**, `backticks for code`, ~~strikethrough~~, and headings.
It ignores random * and correctly handles escaped \*asterisks\* and \_underlines\_ and \`backticks\`. It also supports inline Markdown [Links](http://voyagetravelapps.com/).
> It also now supports blockquotes
> and it supports whole-line italic and bold styles so you can go completely wild with styling! Wow! Such styles! Much fun!
**Lists**
- It Supports
- Unordered
- Lists
- Indented item with a longer string to make sure indentation is consistent
- Second level indent with a longer string to make sure indentation is consistent
- List item with a longer string to make sure indentation is consistent
1. And
1. Ordered
1. Lists
1. Indented item
1. Second level indent
1. (Use `1.` as the list item identifier)
1. List item
1. List item
- Mix
- List styles
1. List item with a longer string to make sure indentation is consistent
1. List item
1. List item
1. List item
1. List item

2
Sources/SwiftyMarkdown/SwiftyMarkdown.swift
View File

@ -188,7 +188,7 @@ If that is not set, then the system default will be used.
CharacterRuleTag(tag: "(", type: .metadataOpen),
CharacterRuleTag(tag: ")", type: .metadataClose)
], styles: [1 : CharacterStyle.link], metadataLookup: false, definesBoundary: true),
CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.code], shouldCancelRemainingTags: true, balancedTags: true),
CharacterRule(primaryTag: CharacterRuleTag(tag: "`", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.code], shouldCancelRemainingRules: true, balancedTags: true),
CharacterRule(primaryTag:CharacterRuleTag(tag: "~", type: .repeating), otherTags : [], styles: [2 : CharacterStyle.strikethrough], minTags:2 , maxTags:2),
CharacterRule(primaryTag: CharacterRuleTag(tag: "*", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2),
CharacterRule(primaryTag: CharacterRuleTag(tag: "_", type: .repeating), otherTags: [], styles: [1 : CharacterStyle.italic, 2 : CharacterStyle.bold], minTags:1 , maxTags:2)

567
Sources/SwiftyMarkdown/SwiftyScanner.swift
View File

@ -1,3 +1,10 @@
//
// SwiftyScanner.swift
//
//
// Created by Simon Fairbairn on 04/04/2020.
//
//
// SwiftyScanner.swift
// SwiftyMarkdown
@ -10,50 +17,548 @@ import os.log
extension OSLog {
private static var subsystem = "SwiftyScanner"
static let swiftyScannerTokenising = OSLog(subsystem: subsystem, category: "Swifty Scanner Tokenising")
static let swiftyScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Peformance")
static let swiftyScanner = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner")
static let swiftyScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner Peformance")
}
/// Swifty Scanning Protocol
public protocol SwiftyScanning {
var metadataLookup : [String : String] { get set }
func scan( _ string : String, with rule : CharacterRule) -> [Token]
func scan( _ tokens : [Token], with rule : CharacterRule) -> [Token]
}
enum TagState {
case none
enum RepeatingTagType {
case open
case intermediate
case closed
case either
case close
case neither
}
class SwiftyScanner : SwiftyScanning {
var metadataLookup: [String : String] = [:]
struct TagGroup {
let groupID = UUID().uuidString
var tagRanges : [ClosedRange<Int>]
var tagType : RepeatingTagType = .open
var count = 1
}
class SwiftyScanner {
var elements : [Element]
let rule : CharacterRule
let metadata : [String : String]
var pointer : Int = 0
init() {
var spaceAndNewLine = CharacterSet.whitespacesAndNewlines
var tagGroups : [TagGroup] = []
var isMetadataOpen = false
var enableLog = (ProcessInfo.processInfo.environment["SwiftyScannerScanner"] != nil)
let currentPerfomanceLog = PerformanceLog(with: "SwiftyScannerScannerPerformanceLogging", identifier: "Scanner", log: OSLog.swiftyScannerPerformance)
let log = PerformanceLog(with: "SwiftyScannerScanner", identifier: "Scanner", log: OSLog.swiftyScanner)
enum Position {
case forward(Int)
case backward(Int)
}
func scan(_ string: String, with rule: CharacterRule) -> [Token] {
return []
init( withElements elements : [Element], rule : CharacterRule, metadata : [String : String]) {
self.elements = elements
self.rule = rule
self.currentPerfomanceLog.start()
self.metadata = metadata
}
func scan(_ tokens: [Token], with rule: CharacterRule) -> [Token] {
return tokens
func elementsBetweenCurrentPosition( and newPosition : Position ) -> [Element]? {
let newIdx : Int
var isForward = true
switch newPosition {
case .backward(let positions):
isForward = false
newIdx = pointer - positions
if newIdx < 0 {
return nil
}
case .forward(let positions):
newIdx = pointer + positions
if newIdx >= self.elements.count {
return nil
}
}
let range : ClosedRange<Int> = ( isForward ) ? self.pointer...newIdx : newIdx...self.pointer
return Array(self.elements[range])
}
func element( for position : Position ) -> Element? {
let newIdx : Int
switch position {
case .backward(let positions):
newIdx = pointer - positions
if newIdx < 0 {
return nil
}
case .forward(let positions):
newIdx = pointer + positions
if newIdx >= self.elements.count {
return nil
}
}
return self.elements[newIdx]
}
func positionIsEqualTo( character : Character, direction : Position ) -> Bool {
guard let validElement = self.element(for: direction) else {
return false
}
return validElement.character == character
}
func positionContains( characters : [Character], direction : Position ) -> Bool {
guard let validElement = self.element(for: direction) else {
return false
}
return characters.contains(validElement.character)
}
func isEscaped() -> Bool {
let isEscaped = self.positionContains(characters: self.rule.escapeCharacters, direction: .backward(1))
if isEscaped {
self.elements[self.pointer - 1].type = .escape
}
return isEscaped
}
func range( for tag : String? ) -> ClosedRange<Int>? {
}
struct TokenGroup {
enum TokenGroupType {
case string
case tag
case escape
guard let tag = tag else {
return nil
}
guard let openChar = tag.first else {
return nil
}
if self.pointer == self.elements.count {
return nil
}
if self.elements[self.pointer].character != openChar {
return nil
}
if isEscaped() {
return nil
}
let range : ClosedRange<Int>
if tag.count > 1 {
guard let elements = self.elementsBetweenCurrentPosition(and: .forward(tag.count - 1) ) else {
return nil
}
// If it's already a tag, then it should be ignored
if elements.filter({ $0.type != .string }).count > 0 {
return nil
}
if elements.map( { String($0.character) }).joined() != tag {
return nil
}
let endIdx = (self.pointer + tag.count - 1)
for i in self.pointer...endIdx {
self.elements[i].type = .tag
}
range = self.pointer...endIdx
self.pointer += tag.count
} else {
// If it's already a tag, then it should be ignored
if self.elements[self.pointer].type != .string {
return nil
}
self.elements[self.pointer].type = .tag
range = self.pointer...self.pointer
self.pointer += 1
}
return range
}
let string : String
let isEscaped : Bool
let type : TokenGroupType
var state : TagState = .none
func resetTagGroup( withID id : String ) {
if let idx = self.tagGroups.firstIndex(where: { $0.groupID == id }) {
for range in self.tagGroups[idx].tagRanges {
self.resetTag(in: range)
}
self.tagGroups.remove(at: idx)
}
self.isMetadataOpen = false
}
func resetTag( in range : ClosedRange<Int>) {
for idx in range {
self.elements[idx].type = .string
}
}
func resetLastTag( for range : inout [ClosedRange<Int>]) {
guard let last = range.last else {
return
}
for idx in last {
self.elements[idx].type = .string
}
}
func closeTag( _ tag : String, withGroupID id : String ) {
guard let tagIdx = self.tagGroups.firstIndex(where: { $0.groupID == id }) else {
return
}
var metadataString = ""
if self.isMetadataOpen {
let metadataCloseRange = self.tagGroups[tagIdx].tagRanges.removeLast()
let metadataOpenRange = self.tagGroups[tagIdx].tagRanges.removeLast()
if metadataOpenRange.upperBound + 1 == (metadataCloseRange.lowerBound) {
if self.enableLog {
os_log("Nothing between the tags", log: OSLog.swiftyScanner, type:.info , self.rule.description)
}
} else {
for idx in (metadataOpenRange.upperBound)...(metadataCloseRange.lowerBound) {
self.elements[idx].type = .metadata
if self.rule.definesBoundary {
self.elements[idx].boundaryCount += 1
}
}
let key = self.elements[metadataOpenRange.upperBound + 1..<metadataCloseRange.lowerBound].map( { String( $0.character )}).joined()
if self.rule.metadataLookup {
metadataString = self.metadata[key] ?? ""
} else {
metadataString = key
}
}
}
let closeRange = self.tagGroups[tagIdx].tagRanges.removeLast()
let openRange = self.tagGroups[tagIdx].tagRanges.removeLast()
if self.rule.balancedTags && closeRange.count != openRange.count {
self.tagGroups[tagIdx].tagRanges.append(openRange)
self.tagGroups[tagIdx].tagRanges.append(closeRange)
return
}
var shouldRemove = true
var styles : [CharacterStyling] = []
if openRange.upperBound + 1 == (closeRange.lowerBound) {
if self.enableLog {
os_log("Nothing between the tags", log: OSLog.swiftyScanner, type:.info , self.rule.description)
}
} else {
var remainingTags = min(openRange.upperBound - openRange.lowerBound, closeRange.upperBound - closeRange.lowerBound) + 1
while remainingTags > 0 {
if remainingTags >= self.rule.maxTags {
remainingTags -= self.rule.maxTags
if let style = self.rule.styles[ self.rule.maxTags ] {
if !styles.contains(where: { $0.isEqualTo(style)}) {
styles.append(style)
}
}
}
if let style = self.rule.styles[remainingTags] {
remainingTags -= remainingTags
if !styles.contains(where: { $0.isEqualTo(style)}) {
styles.append(style)
}
}
}
for idx in (openRange.upperBound)...(closeRange.lowerBound) {
self.elements[idx].styles.append(contentsOf: styles)
self.elements[idx].metadata.append(metadataString)
if self.rule.definesBoundary {
self.elements[idx].boundaryCount += 1
}
if self.rule.shouldCancelRemainingRules {
self.elements[idx].boundaryCount = 1000
}
}
if self.rule.isRepeatingTag {
let difference = ( openRange.upperBound - openRange.lowerBound ) - (closeRange.upperBound - closeRange.lowerBound)
switch difference {
case 1...:
shouldRemove = false
self.tagGroups[tagIdx].count = difference
self.tagGroups[tagIdx].tagRanges.append( openRange.upperBound - (abs(difference) - 1)...openRange.upperBound )
case ...(-1):
for idx in closeRange.upperBound - (abs(difference) - 1)...closeRange.upperBound {
self.elements[idx].type = .string
}
default:
break
}
}
}
if shouldRemove {
self.tagGroups.removeAll(where: { $0.groupID == id })
}
self.isMetadataOpen = false
}
func emptyRanges( _ ranges : inout [ClosedRange<Int>] ) {
while !ranges.isEmpty {
self.resetLastTag(for: &ranges)
ranges.removeLast()
}
}
func scanNonRepeatingTags() {
var groupID = ""
let closeTag = self.rule.tag(for: .close)?.tag
let metadataOpen = self.rule.tag(for: .metadataOpen)?.tag
let metadataClose = self.rule.tag(for: .metadataClose)?.tag
while self.pointer < self.elements.count {
if self.enableLog {
os_log("CHARACTER: %@", log: OSLog.swiftyScanner, type:.info , String(self.elements[self.pointer].character))
}
if let range = self.range(for: metadataClose) {
if self.isMetadataOpen {
guard let groupIdx = self.tagGroups.firstIndex(where: { $0.groupID == groupID }) else {
self.pointer += 1
continue
}
guard !self.tagGroups.isEmpty else {
self.resetTagGroup(withID: groupID)
continue
}
guard self.isMetadataOpen else {
self.resetTagGroup(withID: groupID)
continue
}
if self.enableLog {
os_log("Closing metadata tag found. Closing tag with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
}
self.tagGroups[groupIdx].tagRanges.append(range)
self.closeTag(closeTag!, withGroupID: groupID)
self.isMetadataOpen = false
continue
} else {
self.resetTag(in: range)
self.pointer -= metadataClose!.count
}
}
if let openRange = self.range(for: self.rule.primaryTag.tag) {
if self.isMetadataOpen {
self.resetTagGroup(withID: groupID)
}
let tagGroup = TagGroup(tagRanges: [openRange])
groupID = tagGroup.groupID
if self.enableLog {
os_log("New open tag found. Starting new Group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
}
if self.rule.isRepeatingTag {
}
self.tagGroups.append(tagGroup)
continue
}
if let range = self.range(for: closeTag) {
guard !self.tagGroups.isEmpty else {
if self.enableLog {
os_log("No open tags exist, resetting this close tag", log: OSLog.swiftyScanner, type:.info)
}
self.resetTag(in: range)
continue
}
self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
groupID = self.tagGroups[self.tagGroups.count - 1].groupID
if self.enableLog {
os_log("New close tag found. Appending to group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
}
guard metadataOpen != nil else {
if self.enableLog {
os_log("No metadata tags exist, closing valid tag with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
}
self.closeTag(closeTag!, withGroupID: groupID)
continue
}
guard self.pointer != self.elements.count else {
continue
}
guard let range = self.range(for: metadataOpen) else {
if self.enableLog {
os_log("No metadata tag found, resetting group with ID %@", log: OSLog.swiftyScanner, type:.info , groupID)
}
self.resetTagGroup(withID: groupID)
continue
}
self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
self.isMetadataOpen = true
continue
}
if let range = self.range(for: metadataOpen) {
if self.enableLog {
os_log("Multiple open metadata tags found!", log: OSLog.swiftyScanner, type:.info , groupID)
}
self.resetTag(in: range)
self.resetTagGroup(withID: groupID)
self.isMetadataOpen = false
continue
}
self.pointer += 1
}
}
func scanRepeatingTags() {
var groupID = ""
let escapeCharacters = "" //self.rule.escapeCharacters.map( { String( $0 ) }).joined()
let unionSet = spaceAndNewLine.union(CharacterSet(charactersIn: escapeCharacters))
while self.pointer < self.elements.count {
if self.enableLog {
os_log("CHARACTER: %@", log: OSLog.swiftyScanner, type:.info , String(self.elements[self.pointer].character))
}
if var openRange = self.range(for: self.rule.primaryTag.tag) {
if self.elements[openRange].first?.boundaryCount == 1000 {
self.resetTag(in: openRange)
continue
}
var count = 1
var tagType : RepeatingTagType = .open
if let prevElement = self.element(for: .backward(self.rule.primaryTag.tag.count + 1)) {
if !unionSet.containsUnicodeScalars(of: prevElement.character) {
tagType = .either
}
} else {
tagType = .open
}
while let nextRange = self.range(for: self.rule.primaryTag.tag) {
count += 1
openRange = openRange.lowerBound...nextRange.upperBound
}
if self.rule.minTags > 1 {
if (openRange.upperBound - openRange.lowerBound) + 1 < self.rule.minTags {
self.resetTag(in: openRange)
os_log("Tag does not meet minimum length", log: .swiftyScanner, type: .info)
continue
}
}
var validTagGroup = true
if let nextElement = self.element(for: .forward(0)) {
if unionSet.containsUnicodeScalars(of: nextElement.character) {
if tagType == .either {
tagType = .close
} else {
validTagGroup = tagType != .open
}
}
} else {
if tagType == .either {
tagType = .close
} else {
validTagGroup = tagType != .open
}
}
if !validTagGroup {
if self.enableLog {
os_log("Tag has whitespace on both sides", log: .swiftyScanner, type: .info)
}
self.resetTag(in: openRange)
continue
}
if let idx = tagGroups.firstIndex(where: { $0.groupID == groupID }) {
if tagType == .either {
if tagGroups[idx].count == count {
self.tagGroups[idx].tagRanges.append(openRange)
self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
if let last = self.tagGroups.last {
groupID = last.groupID
}
continue
}
} else {
if let prevRange = tagGroups[idx].tagRanges.first {
if self.elements[prevRange].first?.boundaryCount == self.elements[openRange].first?.boundaryCount {
self.tagGroups[idx].tagRanges.append(openRange)
self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
}
}
continue
}
}
var tagGroup = TagGroup(tagRanges: [openRange])
groupID = tagGroup.groupID
tagGroup.tagType = tagType
tagGroup.count = count
if self.enableLog {
os_log("New open tag found with characters %@. Starting new Group with ID %@", log: OSLog.swiftyScanner, type:.info, self.elements[openRange].map( { String($0.character) }).joined(), groupID)
}
self.tagGroups.append(tagGroup)
continue
}
self.pointer += 1
}
}
func scan() -> [Element] {
guard self.elements.filter({ $0.type == .string }).map({ String($0.character) }).joined().contains(self.rule.primaryTag.tag) else {
return self.elements
}
self.currentPerfomanceLog.tag(with: "Beginning \(self.rule.primaryTag.tag)")
if self.enableLog {
os_log("RULE: %@", log: OSLog.swiftyScanner, type:.info , self.rule.description)
}
if self.rule.isRepeatingTag {
self.scanRepeatingTags()
} else {
self.scanNonRepeatingTags()
}
for tagGroup in self.tagGroups {
self.resetTagGroup(withID: tagGroup.groupID)
}
if self.enableLog {
for element in self.elements {
print(element)
}
}
return self.elements
}
}

565
Sources/SwiftyMarkdown/SwiftyScannerNonRepeating.swift
View File

@ -1,565 +0,0 @@
//
// File.swift
//
//
// Created by Simon Fairbairn on 04/04/2020.
//
//
// SwiftyScanner.swift
// SwiftyMarkdown
//
// Created by Simon Fairbairn on 04/02/2020.
//
import Foundation
import os.log
extension OSLog {
private static var subsystem = "SwiftyScanner"
static let swiftyScannerScanner = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner")
static let swiftyScannerScannerPerformance = OSLog(subsystem: subsystem, category: "Swifty Scanner Scanner Peformance")
}
enum RepeatingTagType {
case open
case either
case close
case neither
}
struct TagGroup {
let groupID = UUID().uuidString
var tagRanges : [ClosedRange<Int>]
var tagType : RepeatingTagType = .open
var count = 1
}
class SwiftyScannerNonRepeating {
var elements : [Element]
let rule : CharacterRule
let metadata : [String : String]
var pointer : Int = 0
var spaceAndNewLine = CharacterSet.whitespacesAndNewlines
var tagGroups : [TagGroup] = []
var isMetadataOpen = false
var enableLog = (ProcessInfo.processInfo.environment["SwiftyScannerScanner"] != nil)
let currentPerfomanceLog = PerformanceLog(with: "SwiftyScannerScannerPerformanceLogging", identifier: "Scanner", log: OSLog.swiftyScannerPerformance)
let log = PerformanceLog(with: "SwiftyScannerScanner", identifier: "Scanner", log: OSLog.swiftyScannerScanner)
enum Position {
case forward(Int)
case backward(Int)
}
init( withElements elements : [Element], rule : CharacterRule, metadata : [String : String]) {
self.elements = elements
self.rule = rule
self.currentPerfomanceLog.start()
self.metadata = metadata
}
func elementsBetweenCurrentPosition( and newPosition : Position ) -> [Element]? {
let newIdx : Int
var isForward = true
switch newPosition {
case .backward(let positions):
isForward = false
newIdx = pointer - positions
if newIdx < 0 {
return nil
}
case .forward(let positions):
newIdx = pointer + positions
if newIdx >= self.elements.count {
return nil
}
}
let range : ClosedRange<Int> = ( isForward ) ? self.pointer...newIdx : newIdx...self.pointer
return Array(self.elements[range])
}
func element( for position : Position ) -> Element? {
let newIdx : Int
switch position {
case .backward(let positions):
newIdx = pointer - positions
if newIdx < 0 {
return nil
}
case .forward(let positions):
newIdx = pointer + positions
if newIdx >= self.elements.count {
return nil
}
}
return self.elements[newIdx]
}
func positionIsEqualTo( character : Character, direction : Position ) -> Bool {
guard let validElement = self.element(for: direction) else {
return false
}
return validElement.character == character
}
func positionContains( characters : [Character], direction : Position ) -> Bool {
guard let validElement = self.element(for: direction) else {
return false
}
return characters.contains(validElement.character)
}
func isEscaped() -> Bool {
let isEscaped = self.positionContains(characters: self.rule.escapeCharacters, direction: .backward(1))
if isEscaped {
self.elements[self.pointer - 1].type = .escape
}
return isEscaped
}
func range( for tag : String? ) -> ClosedRange<Int>? {
guard let tag = tag else {
return nil
}
guard let openChar = tag.first else {
return nil
}
if self.pointer == self.elements.count {
return nil
}
if self.elements[self.pointer].character != openChar {
return nil
}
if isEscaped() {
return nil
}
let range : ClosedRange<Int>
if tag.count > 1 {
guard let elements = self.elementsBetweenCurrentPosition(and: .forward(tag.count - 1) ) else {
return nil
}
// If it's already a tag, then it should be ignored
if elements.filter({ $0.type != .string }).count > 0 {
return nil
}
if elements.map( { String($0.character) }).joined() != tag {
return nil
}
let endIdx = (self.pointer + tag.count - 1)
for i in self.pointer...endIdx {
self.elements[i].type = .tag
}
range = self.pointer...endIdx
self.pointer += tag.count
} else {
// If it's already a tag, then it should be ignored
if self.elements[self.pointer].type != .string {
return nil
}
self.elements[self.pointer].type = .tag
range = self.pointer...self.pointer
self.pointer += 1
}
return range
}
func resetTagGroup( withID id : String ) {
if let idx = self.tagGroups.firstIndex(where: { $0.groupID == id }) {
for range in self.tagGroups[idx].tagRanges {
self.resetTag(in: range)
}
self.tagGroups.remove(at: idx)
}
self.isMetadataOpen = false
}
func resetTag( in range : ClosedRange<Int>) {
for idx in range {
self.elements[idx].type = .string
}
}
func resetLastTag( for range : inout [ClosedRange<Int>]) {
guard let last = range.last else {
return
}
for idx in last {
self.elements[idx].type = .string
}
}
func closeTag( _ tag : String, withGroupID id : String ) {
guard let tagIdx = self.tagGroups.firstIndex(where: { $0.groupID == id }) else {
return
}
var metadataString = ""
if self.isMetadataOpen {
let metadataCloseRange = self.tagGroups[tagIdx].tagRanges.removeLast()
let metadataOpenRange = self.tagGroups[tagIdx].tagRanges.removeLast()
if metadataOpenRange.upperBound + 1 == (metadataCloseRange.lowerBound) {
if self.enableLog {
os_log("Nothing between the tags", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
}
} else {
for idx in (metadataOpenRange.upperBound)...(metadataCloseRange.lowerBound) {
self.elements[idx].type = .metadata
if self.rule.definesBoundary {
self.elements[idx].boundaryCount += 1
}
}
let key = self.elements[metadataOpenRange.upperBound + 1..<metadataCloseRange.lowerBound].map( { String( $0.character )}).joined()
if self.rule.metadataLookup {
metadataString = self.metadata[key] ?? ""
} else {
metadataString = key
}
}
}
let closeRange = self.tagGroups[tagIdx].tagRanges.removeLast()
let openRange = self.tagGroups[tagIdx].tagRanges.removeLast()
if self.rule.balancedTags && closeRange.count != openRange.count {
self.tagGroups[tagIdx].tagRanges.append(openRange)
self.tagGroups[tagIdx].tagRanges.append(closeRange)
return
}
var shouldRemove = true
var styles : [CharacterStyling] = []
if openRange.upperBound + 1 == (closeRange.lowerBound) {
if self.enableLog {
os_log("Nothing between the tags", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
}
} else {
var remainingTags = min(openRange.upperBound - openRange.lowerBound, closeRange.upperBound - closeRange.lowerBound) + 1
while remainingTags > 0 {
if remainingTags >= self.rule.maxTags {
remainingTags -= self.rule.maxTags
if let style = self.rule.styles[ self.rule.maxTags ] {
if !styles.contains(where: { $0.isEqualTo(style)}) {
styles.append(style)
}
}
}
if let style = self.rule.styles[remainingTags] {
remainingTags -= remainingTags
if !styles.contains(where: { $0.isEqualTo(style)}) {
styles.append(style)
}
}
}
for idx in (openRange.upperBound)...(closeRange.lowerBound) {
self.elements[idx].styles.append(contentsOf: styles)
self.elements[idx].metadata.append(metadataString)
if self.rule.definesBoundary {
self.elements[idx].boundaryCount += 1
}
if self.rule.shouldCancelRemainingRules {
self.elements[idx].boundaryCount = 1000
}
}
if self.rule.isRepeatingTag {
let difference = ( openRange.upperBound - openRange.lowerBound ) - (closeRange.upperBound - closeRange.lowerBound)
switch difference {
case 1...:
shouldRemove = false
self.tagGroups[tagIdx].count = difference
self.tagGroups[tagIdx].tagRanges.append( openRange.upperBound - (abs(difference) - 1)...openRange.upperBound )
case ...(-1):
for idx in closeRange.upperBound - (abs(difference) - 1)...closeRange.upperBound {
self.elements[idx].type = .string
}
default:
break
}
}
}
if shouldRemove {
self.tagGroups.removeAll(where: { $0.groupID == id })
}
self.isMetadataOpen = false
}
func emptyRanges( _ ranges : inout [ClosedRange<Int>] ) {
while !ranges.isEmpty {
self.resetLastTag(for: &ranges)
ranges.removeLast()
}
}
func scanNonRepeatingTags() {
var groupID = ""
let closeTag = self.rule.tag(for: .close)?.tag
let metadataOpen = self.rule.tag(for: .metadataOpen)?.tag
let metadataClose = self.rule.tag(for: .metadataClose)?.tag
while self.pointer < self.elements.count {
if self.enableLog {
os_log("CHARACTER: %@", log: OSLog.swiftyScannerScanner, type:.info , String(self.elements[self.pointer].character))
}
if let range = self.range(for: metadataClose) {
if self.isMetadataOpen {
guard let groupIdx = self.tagGroups.firstIndex(where: { $0.groupID == groupID }) else {
self.pointer += 1
continue
}
guard !self.tagGroups.isEmpty else {
self.resetTagGroup(withID: groupID)
continue
}
guard self.isMetadataOpen else {
self.resetTagGroup(withID: groupID)
continue
}
if self.enableLog {
os_log("Closing metadata tag found. Closing tag with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
}
self.tagGroups[groupIdx].tagRanges.append(range)
self.closeTag(closeTag!, withGroupID: groupID)
self.isMetadataOpen = false
continue
} else {
self.resetTag(in: range)
self.pointer -= metadataClose!.count
}
}
if let openRange = self.range(for: self.rule.primaryTag.tag) {
if self.isMetadataOpen {
self.resetTagGroup(withID: groupID)
}
let tagGroup = TagGroup(tagRanges: [openRange])
groupID = tagGroup.groupID
if self.enableLog {
os_log("New open tag found. Starting new Group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
}
if self.rule.isRepeatingTag {
}
self.tagGroups.append(tagGroup)
continue
}
if let range = self.range(for: closeTag) {
guard !self.tagGroups.isEmpty else {
if self.enableLog {
os_log("No open tags exist, resetting this close tag", log: OSLog.swiftyScannerScanner, type:.info)
}
self.resetTag(in: range)
continue
}
self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
groupID = self.tagGroups[self.tagGroups.count - 1].groupID
if self.enableLog {
os_log("New close tag found. Appending to group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
}
guard metadataOpen != nil else {
if self.enableLog {
os_log("No metadata tags exist, closing valid tag with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
}
self.closeTag(closeTag!, withGroupID: groupID)
continue
}
guard self.pointer != self.elements.count else {
continue
}
guard let range = self.range(for: metadataOpen) else {
if self.enableLog {
os_log("No metadata tag found, resetting group with ID %@", log: OSLog.swiftyScannerScanner, type:.info , groupID)
}
self.resetTagGroup(withID: groupID)
continue
}
self.tagGroups[self.tagGroups.count - 1].tagRanges.append(range)
self.isMetadataOpen = true
continue
}
if let range = self.range(for: metadataOpen) {
if self.enableLog {
os_log("Multiple open metadata tags found!", log: OSLog.swiftyScannerScanner, type:.info , groupID)
}
self.resetTag(in: range)
self.resetTagGroup(withID: groupID)
self.isMetadataOpen = false
continue
}
self.pointer += 1
}
}
func scanRepeatingTags() {
var groupID = ""
let escapeCharacters = "" //self.rule.escapeCharacters.map( { String( $0 ) }).joined()
let unionSet = spaceAndNewLine.union(CharacterSet(charactersIn: escapeCharacters))
while self.pointer < self.elements.count {
if self.enableLog {
os_log("CHARACTER: %@", log: OSLog.swiftyScannerScanner, type:.info , String(self.elements[self.pointer].character))
}
if var openRange = self.range(for: self.rule.primaryTag.tag) {
if self.elements[openRange].first?.boundaryCount == 1000 {
self.resetTag(in: openRange)
continue
}
var count = 1
var tagType : RepeatingTagType = .open
if let prevElement = self.element(for: .backward(self.rule.primaryTag.tag.count + 1)) {
if !unionSet.containsUnicodeScalars(of: prevElement.character) {
tagType = .either
}
} else {
tagType = .open
}
while let nextRange = self.range(for: self.rule.primaryTag.tag) {
count += 1
openRange = openRange.lowerBound...nextRange.upperBound
}
if self.rule.minTags > 1 {
if (openRange.upperBound - openRange.lowerBound) + 1 < self.rule.minTags {
self.resetTag(in: openRange)
os_log("Tag does not meet minimum length", log: .swiftyScannerScanner, type: .info)
continue
}
}
var validTagGroup = true
if let nextElement = self.element(for: .forward(0)) {
if unionSet.containsUnicodeScalars(of: nextElement.character) {
if tagType == .either {
tagType = .close
} else {
validTagGroup = tagType != .open
}
}
} else {
if tagType == .either {
tagType = .close
} else {
validTagGroup = tagType != .open
}
}
if !validTagGroup {
if self.enableLog {
os_log("Tag has whitespace on both sides", log: .swiftyScannerScanner, type: .info)
}
self.resetTag(in: openRange)
continue
}
if let idx = tagGroups.firstIndex(where: { $0.groupID == groupID }) {
if tagType == .either {
if tagGroups[idx].count == count {
self.tagGroups[idx].tagRanges.append(openRange)
self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
if let last = self.tagGroups.last {
groupID = last.groupID
}
continue
}
} else {
if let prevRange = tagGroups[idx].tagRanges.first {
if self.elements[prevRange].first?.boundaryCount == self.elements[openRange].first?.boundaryCount {
self.tagGroups[idx].tagRanges.append(openRange)
self.closeTag(self.rule.primaryTag.tag, withGroupID: groupID)
}
}
continue
}
}
var tagGroup = TagGroup(tagRanges: [openRange])
groupID = tagGroup.groupID
tagGroup.tagType = tagType
tagGroup.count = count
if self.enableLog {
os_log("New open tag found with characters %@. Starting new Group with ID %@", log: OSLog.swiftyScannerScanner, type:.info, self.elements[openRange].map( { String($0.character) }).joined(), groupID)
}
self.tagGroups.append(tagGroup)
continue
}
self.pointer += 1
}
}
func scan() -> [Element] {
guard self.elements.filter({ $0.type == .string }).map({ String($0.character) }).joined().contains(self.rule.primaryTag.tag) else {
return self.elements
}
self.currentPerfomanceLog.tag(with: "Beginning \(self.rule.primaryTag.tag)")
if self.enableLog {
os_log("RULE: %@", log: OSLog.swiftyScannerScanner, type:.info , self.rule.description)
}
if self.rule.isRepeatingTag {
self.scanRepeatingTags()
} else {
self.scanNonRepeatingTags()
}
for tagGroup in self.tagGroups {
self.resetTagGroup(withID: tagGroup.groupID)
}
if self.enableLog {
for element in self.elements {
print(element)
}
}
return self.elements
}
}

378
Sources/SwiftyMarkdown/SwiftyTokeniser.swift
View File

@ -23,7 +23,6 @@ public class SwiftyTokeniser {
let totalPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Total Run Time", log: OSLog.performance)
let currentPerfomanceLog = PerformanceLog(with: "SwiftyTokeniserPerformanceLogging", identifier: "Tokeniser Current", log: OSLog.performance)
var scanner : SwiftyScanning!
public var metadataLookup : [String : String] = [:]
let newlines = CharacterSet.newlines
@ -53,7 +52,7 @@ public class SwiftyTokeniser {
///
/// - Parameter inputString: A string to have the CharacterRules in `self.rules` applied to
public func process( _ inputString : String ) -> [Token] {
var currentTokens = [Token(type: .string, inputString: inputString)]
let currentTokens = [Token(type: .string, inputString: inputString)]
guard rules.count > 0 else {
return currentTokens
}
@ -83,19 +82,13 @@ public class SwiftyTokeniser {
while !mutableRules.isEmpty {
let nextRule = mutableRules.removeFirst()
if nextRule.isRepeatingTag {
self.scanner = SwiftyScanner()
self.scanner.metadataLookup = self.metadataLookup
}
if enableLog {
os_log("------------------------------", log: .tokenising, type: .info)
os_log("RULE: %@", log: OSLog.tokenising, type:.info , nextRule.description)
}
self.currentPerfomanceLog.tag(with: "(start rule %@)")
let scanner = SwiftyScannerNonRepeating(withElements: elementArray, rule: nextRule, metadata: self.metadataLookup)
let scanner = SwiftyScanner(withElements: elementArray, rule: nextRule, metadata: self.metadataLookup)
elementArray = scanner.scan()
}
@ -113,7 +106,6 @@ public class SwiftyTokeniser {
tokens.append(token)
}
var lastElement = elementArray.first!
var accumulatedString = ""
for element in elementArray {
@ -141,372 +133,6 @@ public class SwiftyTokeniser {
}
return output
}
//
//
// /// In order to reinsert the original replacements into the new string token, the replacements
// /// need to be searched for in the incoming string one by one.
// ///
// /// Using the `newToken(fromSubstring:isReplacement:)` function ensures that any metadata and character styles
// /// are passed over into the newly created tokens.
// ///
// /// E.g. A string token that has an `outputString` of "This string AAAAA-BBBBB-CCCCC replacements", with
// /// a characterStyle of `bold` for the entire string, needs to be separated into the following tokens:
// ///
// /// - `string`: "This string "
// /// - `replacement`: "AAAAA-BBBBB-CCCCC"
// /// - `string`: " replacements"
// ///
// /// Each of these need to have a character style of `bold`.
// ///
// /// - Parameters:
// /// - replacements: An array of `replacement` tokens
// /// - token: The new `string` token that may contain replacement IDs contained in the `replacements` array
// func reinsertReplacements(_ replacements : [Token], from stringToken : Token ) -> [Token] {
// guard !stringToken.outputString.isEmpty && !replacements.isEmpty else {
// return [stringToken]
// }
// var outputTokens : [Token] = []
// let scanner = Scanner(string: stringToken.outputString)
// scanner.charactersToBeSkipped = nil
//
// // Remove any replacements that don't appear in the incoming string
// var repTokens = replacements.filter({ stringToken.outputString.contains($0.inputString) })
//
// var testString = "\n"
// while !scanner.isAtEnd {
// var outputString : String = ""
// if repTokens.count > 0 {
// testString = repTokens.removeFirst().inputString
// }
//
// if #available(iOS 13.0, OSX 10.15, watchOS 6.0, tvOS 13.0, *) {
// if let nextString = scanner.scanUpToString(testString) {
// outputString = nextString
// outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
// if let outputToken = scanner.scanString(testString) {
// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
// }
// } else if let outputToken = scanner.scanString(testString) {
// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
// }
// } else {
// var oldString : NSString? = nil
// var tokenString : NSString? = nil
// scanner.scanUpTo(testString, into: &oldString)
// if let nextString = oldString {
// outputString = nextString as String
// outputTokens.append(stringToken.newToken(fromSubstring: outputString, isReplacement: false))
// scanner.scanString(testString, into: &tokenString)
// if let outputToken = tokenString as String? {
// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
// }
// } else {
// scanner.scanString(testString, into: &tokenString)
// if let outputToken = tokenString as String? {
// outputTokens.append(stringToken.newToken(fromSubstring: outputToken, isReplacement: true))
// }
// }
// }
// }
// return outputTokens
// }
//
//
// /// This function is necessary because a previously tokenised string might have
// ///
// /// Consider a previously tokenised string, where AAAAA-BBBBB-CCCCC represents a replaced \[link\](url) instance.
// ///
// /// The incoming tokens will look like this:
// ///
// /// - `string`: "A \*\*Bold"
// /// - `replacement` : "AAAAA-BBBBB-CCCCC"
// /// - `string`: " with a trailing string**"
// ///
// /// However, because the scanner can only tokenise individual strings, passing in the string values
// /// of these tokens individually and applying the styles will not correctly detect the starting and
// /// ending `repeatingTag` instances. (e.g. the scanner will see "A \*\*Bold", and then "AAAAA-BBBBB-CCCCC",
// /// and finally " with a trailing string\*\*")
// ///
// /// The strings need to be combined, so that they form a single string:
// /// A \*\*Bold AAAAA-BBBBB-CCCCC with a trailing string\*\*.
// /// This string is then parsed and tokenised so that it looks like this:
// ///
// /// - `string`: "A "
// /// - `repeatingTag`: "\*\*"
// /// - `string`: "Bold AAAAA-BBBBB-CCCCC with a trailing string"
// /// - `repeatingTag`: "\*\*"
// ///
// /// Finally, the replacements from the original incoming token array are searched for and pulled out
// /// of this new string, so the final result looks like this:
// ///
// /// - `string`: "A "
// /// - `repeatingTag`: "\*\*"
// /// - `string`: "Bold "
// /// - `replacement`: "AAAAA-BBBBB-CCCCC"
// /// - `string`: " with a trailing string"
// /// - `repeatingTag`: "\*\*"
// ///
// /// - Parameters:
// /// - tokens: The tokens to be combined, scanned, re-tokenised, and merged
// /// - rule: The character rule currently being applied
// func scanReplacementTokens( _ tokens : [Token], with rule : CharacterRule ) -> [Token] {
// guard tokens.count > 0 else {
// return []
// }
//
// let combinedString = tokens.map({ $0.outputString }).joined()
//
// let nextTokens = self.scanner.scan(combinedString, with: rule)
// var replacedTokens = self.applyStyles(to: nextTokens, usingRule: rule)
//
// /// It's necessary here to check to see if the first token (which will always represent the styles
// /// to be applied from previous scans) has any existing metadata or character styles and apply them
// /// to *all* the string and replacement tokens found by the new scan.
// for idx in 0..<replacedTokens.count {
// guard replacedTokens[idx].type == .string || replacedTokens[idx].type == .replacement else {
// continue
// }
// if tokens.first!.metadataString != nil && replacedTokens[idx].metadataString == nil {
// replacedTokens[idx].metadataString = tokens.first!.metadataString
// }
// replacedTokens[idx].characterStyles.append(contentsOf: tokens.first!.characterStyles)
// }
//
// // Swap the original replacement tokens back in
// let replacements = tokens.filter({ $0.type == .replacement })
// var outputTokens : [Token] = []
// for token in replacedTokens {
// guard token.type == .string else {
// outputTokens.append(token)
// continue
// }
// outputTokens.append(contentsOf: self.reinsertReplacements(replacements, from: token))
// }
//
// return outputTokens
// }
//
//
//
// /// This function ensures that only concurrent `string` and `replacement` tokens are processed together.
// ///
// /// i.e. If there is an existing `repeatingTag` token between two strings, then those strings will be
// /// processed individually. This prevents incorrect parsing of strings like "\*\*\_Should only be bold\*\*\_"
// ///
// /// - Parameters:
// /// - incomingTokens: A group of tokens whose string tokens and replacement tokens should be combined and re-tokenised
// /// - rule: The current rule being processed
// func handleReplacementTokens( _ incomingTokens : [Token], with rule : CharacterRule) -> [Token] {
//
// // Only combine string and replacements that are next to each other.
// var newTokenSet : [Token] = []
// var currentTokenSet : [Token] = []
// for i in 0..<incomingTokens.count {
// guard incomingTokens[i].type == .string || incomingTokens[i].type == .replacement else {
// newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
// newTokenSet.append(incomingTokens[i])
// currentTokenSet.removeAll()
// continue
// }
// guard !incomingTokens[i].isProcessed && !incomingTokens[i].isMetadata && !incomingTokens[i].shouldSkip else {
// newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
// newTokenSet.append(incomingTokens[i])
// currentTokenSet.removeAll()
// continue
// }
// currentTokenSet.append(incomingTokens[i])
// }
// newTokenSet.append(contentsOf: self.scanReplacementTokens(currentTokenSet, with: rule))
//
// return newTokenSet
// }
//
//
// func handleClosingTagFromOpenTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule ) {
//
// guard rule.closeTag != nil else {
// return
// }
// guard let closeTokenIdx = tokens.firstIndex(where: { $0.type == .closeTag && !$0.isProcessed }) else {
// return
// }
//
// var metadataIndex = index
// // If there's an intermediate tag, get the index of that
// if rule.intermediateTag != nil {
// guard let nextTokenIdx = tokens.firstIndex(where: { $0.type == .intermediateTag && !$0.isProcessed }) else {
// return
// }
// metadataIndex = nextTokenIdx
// let styles : [CharacterStyling] = rule.styles[1] ?? []
// for i in index..<nextTokenIdx {
// for style in styles {
// if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
// tokens[i].characterStyles.append(style)
// }
// }
// }
// }
//
// var metadataString : String = ""
// for i in metadataIndex..<closeTokenIdx {
// if tokens[i].type == .string {
// metadataString.append(tokens[i].outputString)
// tokens[i].isMetadata = true
// }
// }
//
// for i in index..<metadataIndex {
// if tokens[i].type == .string {
// tokens[i].metadataString = metadataString
// }
// }
//
// tokens[closeTokenIdx].isProcessed = true
// tokens[metadataIndex].isProcessed = true
// tokens[index].isProcessed = true
// }
//
//
// /// This is here to manage how opening tags are matched with closing tags when they're all the same
// /// character.
// ///
// /// Of course, because Markdown is about as loose as a spec can be while still being considered any
// /// kind of spec, the number of times this character repeats causes different effects. Then there
// /// is the ill-defined way it should work if the number of opening and closing tags are different.
// ///
// /// - Parameters:
// /// - index: The index of the current token in the loop
// /// - tokens: An inout variable of the loop tokens of interest
// /// - rule: The character rule being applied
// func handleClosingTagFromRepeatingTag(withIndex index : Int, in tokens: inout [Token], following rule : CharacterRule) {
// let theToken = tokens[index]
//
// if enableLog {
// os_log("Found repeating tag with tag count: %i, tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString, rule.openTag )
// }
//
// guard theToken.count > 0 else {
// return
// }
//
// let startIdx = index
// var endIdx : Int? = nil
//
// let maxCount = (theToken.count > rule.maxTags) ? rule.maxTags : theToken.count
// // Try to find exact match first
// if let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count == theToken.count && $0.id != theToken.id && !$0.isProcessed && $0.group != theToken.group }) {
// endIdx = nextTokenIdx
// }
//
// if endIdx == nil, let nextTokenIdx = tokens.firstIndex(where: { $0.inputString.first == theToken.inputString.first && $0.type == theToken.type && $0.count >= 1 && $0.id != theToken.id && !$0.isProcessed }) {
// endIdx = nextTokenIdx
// }
// guard let existentEnd = endIdx else {
// return
// }
//
//
// let styles : [CharacterStyling] = rule.styles[maxCount] ?? []
// for i in startIdx..<existentEnd {
// for style in styles {
// if !tokens[i].characterStyles.contains(where: { $0.isEqualTo(style )}) {
// tokens[i].characterStyles.append(style)
// }
// }
// if rule.cancels == .allRemaining {
// tokens[i].shouldSkip = true
// }
// }
//
// let maxEnd = (tokens[existentEnd].count > rule.maxTags) ? rule.maxTags : tokens[existentEnd].count
// tokens[index].count = theToken.count - maxEnd
// tokens[existentEnd].count = tokens[existentEnd].count - maxEnd
// if maxEnd < rule.maxTags {
// self.handleClosingTagFromRepeatingTag(withIndex: index, in: &tokens, following: rule)
// } else {
// tokens[existentEnd].isProcessed = true
// tokens[index].isProcessed = true
// }
//
//
// }
//
// func applyStyles( to tokens : [Token], usingRule rule : CharacterRule ) -> [Token] {
// var mutableTokens : [Token] = tokens
//
// if enableLog {
// os_log("Applying styles to tokens: %@", log: .tokenising, type: .info, tokens.oslogDisplay )
// }
// for idx in 0..<mutableTokens.count {
// let token = mutableTokens[idx]
// switch token.type {
// case .escape:
// if enableLog {
// os_log("Found escape: %@", log: .tokenising, type: .info, token.inputString )
// }
// case .repeatingTag:
// let theToken = mutableTokens[idx]
// self.handleClosingTagFromRepeatingTag(withIndex: idx, in: &mutableTokens, following: rule)
// if enableLog {
// os_log("Found repeating tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
// }
// case .openTag:
// let theToken = mutableTokens[idx]
// if enableLog {
// os_log("Found open tag with tags: %@, current rule open tag: %@", log: .tokenising, type: .info, theToken.inputString, rule.openTag )
// }
//
// guard rule.closingTag != nil else {
//
// // If there's an intermediate tag, get the index of that
//
// // Get the index of the closing tag
//
// continue
// }
// self.handleClosingTagFromOpenTag(withIndex: idx, in: &mutableTokens, following: rule)
//
//
// case .intermediateTag:
// let theToken = mutableTokens[idx]
// if enableLog {
// os_log("Found intermediate tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
// }
//
// case .closeTag:
// let theToken = mutableTokens[idx]
// if enableLog {
// os_log("Found close tag with tag count: %i, tags: %@", log: .tokenising, type: .info, theToken.count, theToken.inputString )
// }
//
// case .string:
// let theToken = mutableTokens[idx]
// if enableLog {
// if theToken.isMetadata {
// os_log("Found Metadata: %@", log: .tokenising, type: .info, theToken.inputString )
// } else {
// os_log("Found String: %@", log: .tokenising, type: .info, theToken.inputString )
// }
// if let hasMetadata = theToken.metadataString {
// os_log("...with metadata: %@", log: .tokenising, type: .info, hasMetadata )
// }
// }
//
// case .replacement:
// if enableLog {
// os_log("Found replacement with ID: %@", log: .tokenising, type: .info, mutableTokens[idx].inputString )
// }
// }
// }
// return mutableTokens
// }
//
//
//
}