Cuckoo/Generator/Source/CuckooGeneratorFramework/Tokenizer.swift

236 lines
10 KiB
Swift

//
// Tokenizer.swift
// CuckooGenerator
//
// Created by Tadeas Kriz on 12/01/16.
// Copyright © 2016 Brightify. All rights reserved.
//
import SourceKittenFramework
public struct Tokenizer {
private let file: File
private let source: String
public init(sourceFile: File) {
self.file = sourceFile
source = sourceFile.contents
}
public func tokenize() -> FileRepresentation {
let structure = Structure(file: file)
let declarations = tokenize(structure.dictionary[Key.Substructure.rawValue] as? [SourceKitRepresentable] ?? [])
let imports = tokenizeImports(declarations)
return FileRepresentation(sourceFile: file, declarations: declarations + imports)
}
private func tokenize(representables: [SourceKitRepresentable]) -> [Token] {
return representables.flatMap(tokenize)
}
private func tokenize(representable: SourceKitRepresentable) -> Token? {
guard let dictionary = representable as? [String: SourceKitRepresentable] else { return nil }
// Common fields
let name = dictionary[Key.Name.rawValue] as? String ?? "name not set"
let kind = dictionary[Key.Kind.rawValue] as? String ?? dictionary[Key.Attribute.rawValue] as? String ?? "unknown type"
// Optional fields
let range = extractRange(dictionary, offsetKey: .Offset, lengthKey: .Length)
let nameRange = extractRange(dictionary, offsetKey: .NameOffset, lengthKey: .NameLength)
let bodyRange = extractRange(dictionary, offsetKey: .BodyOffset, lengthKey: .BodyLength)
let accessibility = (dictionary[Key.Accessibility.rawValue] as? String).flatMap { Accessibility(rawValue: $0) }
let type = dictionary[Key.TypeName.rawValue] as? String
switch kind {
case Kinds.ProtocolDeclaration.rawValue:
let subtokens = tokenize(dictionary[Key.Substructure.rawValue] as? [SourceKitRepresentable] ?? [])
let initializers = subtokens.only(Initializer)
let children = subtokens.noneOf(Initializer)
return ProtocolDeclaration(
name: name,
accessibility: accessibility!,
range: range!,
nameRange: nameRange!,
bodyRange: bodyRange!,
initializers: initializers,
children: children)
case Kinds.ClassDeclaration.rawValue:
let subtokens = tokenize(dictionary[Key.Substructure.rawValue] as? [SourceKitRepresentable] ?? [])
let initializers = subtokens.only(Initializer)
let children = subtokens.noneOf(Initializer).map { child -> Token in
if var property = child as? InstanceVariable {
property.overriding = true
return property
} else {
return child
}
}
return ClassDeclaration(
name: name,
accessibility: accessibility!,
range: range!,
nameRange: nameRange!,
bodyRange: bodyRange!,
initializers: initializers,
children: children)
case Kinds.ExtensionDeclaration.rawValue:
return ExtensionDeclaration(range: range!)
case Kinds.InstanceVariable.rawValue:
let setterAccessibility = (dictionary[Key.SetterAccessibility.rawValue] as? String).flatMap(Accessibility.init)
if String(source.utf8.dropFirst(range!.startIndex)).takeUntilStringOccurs(name)?.trimmed.hasPrefix("let") == true {
return nil
}
return InstanceVariable(
name: name,
type: type!,
accessibility: accessibility!,
setterAccessibility: setterAccessibility,
range: range!,
nameRange: nameRange!,
overriding: false)
case Kinds.InstanceMethod.rawValue:
let parameters = tokenizeMethodParameters(name, dictionary[Key.Substructure.rawValue] as? [SourceKitRepresentable] ?? [])
var returnSignature: String
if let bodyRange = bodyRange {
returnSignature = source[nameRange!.endIndex..<bodyRange.startIndex].takeUntilStringOccurs("{")?.trimmed ?? ""
} else {
returnSignature = source[nameRange!.endIndex..<range!.endIndex].trimmed
if returnSignature.isEmpty {
let untilThrows = String(source.utf8.dropFirst(nameRange!.endIndex))?
.takeUntilStringOccurs("throws").map { $0 + "throws" }?
.trimmed
if let untilThrows = untilThrows where untilThrows == "throws" || untilThrows == "rethrows" {
returnSignature = "\(untilThrows)"
}
}
}
if returnSignature.isEmpty == false {
returnSignature = " " + returnSignature
}
// When bodyRange != nil, we need to create .ClassMethod instead of .ProtocolMethod
if let bodyRange = bodyRange {
return ClassMethod(
name: name,
accessibility: accessibility!,
returnSignature: returnSignature,
range: range!,
nameRange: nameRange!,
parameters: parameters,
bodyRange: bodyRange)
} else {
return ProtocolMethod(
name: name,
accessibility: accessibility!,
returnSignature: returnSignature,
range: range!,
nameRange: nameRange!,
parameters: parameters)
}
case Kinds.Mark.rawValue:
// Do not log warning
return nil
default:
fputs("Unknown kind: \(kind)", stderr)
return nil
}
}
private func tokenizeMethodParameters(methodName: String, _ representables: [SourceKitRepresentable]) -> [MethodParameter] {
// Takes the string between `(` and `)`
let parameters = methodName.componentsSeparatedByString("(").last?.characters.dropLast(1).map { "\($0)" }.joinWithSeparator("")
let parameterLabels: [String?] = parameters?.componentsSeparatedByString(":").map { $0 != "_" ? $0 : nil } ?? []
return zip(parameterLabels, representables).flatMap(tokenizeMethodParameter)
}
private func tokenizeMethodParameter(label: String?, _ representable: SourceKitRepresentable) -> MethodParameter? {
guard let dictionary = representable as? [String: SourceKitRepresentable] else { return nil }
let name = dictionary[Key.Name.rawValue] as? String ?? "name not set"
let kind = dictionary[Key.Kind.rawValue] as? String ?? dictionary[Key.Attribute.rawValue] as? String ?? "unknown type"
let range = extractRange(dictionary, offsetKey: .Offset, lengthKey: .Length)
let nameRange = extractRange(dictionary, offsetKey: .NameOffset, lengthKey: .NameLength)
let type = dictionary[Key.TypeName.rawValue] as? String
switch kind {
case Kinds.MethodParameter.rawValue:
let attributes = tokenizeAttributes(dictionary[Key.Attributes.rawValue] as? [SourceKitRepresentable] ?? [])
return MethodParameter(label: label, name: name, type: type!, range: range!, nameRange: nameRange!, attributes: attributes)
default:
fputs("Unknown method parameter kind: \(kind)", stderr)
return nil
}
}
private func tokenizeAttributes(representables: [SourceKitRepresentable]) -> Attributes {
return representables.map(tokenizeAttribute).reduce(Attributes.none) { $0.union($1) }
}
private func tokenizeAttribute(representable: SourceKitRepresentable) -> Attributes {
guard let dictionary = representable as? [String: SourceKitRepresentable] else { return Attributes.none }
let kind = dictionary[Key.Kind.rawValue] as? String ?? dictionary[Key.Attribute.rawValue] as? String ?? "unknown type"
let range = extractRange(dictionary, offsetKey: .Offset, lengthKey: .Length)
switch kind {
case Kinds.AutoclosureAttribute.rawValue:
let autoclosure = "@autoclosure" + source[0..<range!.startIndex].componentsSeparatedByString("@autoclosure").last!
let escaping = autoclosure.containsString("escaping")
return escaping ? Attributes.escapingAutoclosure : Attributes.autoclosure
case Kinds.NoescapeAttribute.rawValue:
return Attributes.noescape
default:
fputs("Unknown attribute kind: \(kind)", stderr)
return Attributes.none
}
}
private func tokenizeImports(otherTokens: [Token]) -> [Token] {
let rangesToIgnore: [Range<Int>] = otherTokens.flatMap { token in
switch token {
case let container as ContainerToken:
return container.range
case let extensionToken as ExtensionDeclaration:
return extensionToken.range
default:
return nil
}
}
do {
let regex = try NSRegularExpression(pattern: "(?:\\b|;)import(?:\\s|(?:\\/\\/.*\\n)|(?:\\/\\*.*\\*\\/))+([^\\s;\\/]+)", options: [])
let results = regex.matchesInString(source, options: [], range: NSMakeRange(0, source.characters.count))
return results.filter { result in
rangesToIgnore.filter { $0 ~= result.range.location }.isEmpty
}.map {
// This NSRange is always a valid Range
let range = $0.range.toRange()!
let library = (source as NSString).substringWithRange($0.rangeAtIndex(1))
return Import(range: range, library: library)
}
} catch let error as NSError {
fatalError("Invalid regex:" + error.description)
}
}
}