Merge pull request #14 from ktoso/benchmarks
+simple benchmarks to have a gut feeling how mutating and the CoW cos…
This commit is contained in:
Konrad `ktoso` Malawski
GitHub
commit
c78c490045
|
|
@ -11,11 +11,30 @@ let package = Package(
|
|||
name: "Baggage",
|
||||
dependencies: []
|
||||
),
|
||||
|
||||
// ==== --------------------------------------------------------------------------------------------------------
|
||||
// MARK: Tests
|
||||
|
||||
.testTarget(
|
||||
name: "BaggageTests",
|
||||
dependencies: [
|
||||
"Baggage"
|
||||
]
|
||||
)
|
||||
),
|
||||
|
||||
// ==== --------------------------------------------------------------------------------------------------------
|
||||
// MARK: Performance / Benchmarks
|
||||
|
||||
.target(
|
||||
name: "Benchmarks",
|
||||
dependencies: [
|
||||
"Baggage",
|
||||
"SwiftBenchmarkTools",
|
||||
]
|
||||
),
|
||||
.target(
|
||||
name: "SwiftBenchmarkTools",
|
||||
dependencies: []
|
||||
),
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,189 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift Baggage Context open source project
|
||||
//
|
||||
// Copyright (c) 2020 Moritz Lang and the Swift Baggage Context project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import Baggage
|
||||
import Dispatch
|
||||
import class Foundation.NSLock
|
||||
import SwiftBenchmarkTools
|
||||
public let BaggagePassingBenchmarks: [BenchmarkInfo] = [
|
||||
// ==== ----------------------------------------------------------------------------------------------------------------
|
||||
// MARK: "Read only" context passing around
|
||||
BenchmarkInfo(
|
||||
name: "BaggagePassingBenchmarks.pass_async_empty_100_000 ",
|
||||
runFunction: { _ in
|
||||
let context = BaggageContext()
|
||||
pass_async(context: context, times: 100_000)
|
||||
},
|
||||
tags: [],
|
||||
setUpFunction: { setUp() },
|
||||
tearDownFunction: tearDown
|
||||
),
|
||||
BenchmarkInfo(
|
||||
name: "BaggagePassingBenchmarks.pass_async_smol_100_000 ",
|
||||
runFunction: { _ in
|
||||
var context = BaggageContext()
|
||||
context.k1 = "one"
|
||||
context.k2 = "two"
|
||||
context.k3 = "three"
|
||||
context.k4 = "four"
|
||||
pass_async(context: context, times: 100_000)
|
||||
},
|
||||
tags: [],
|
||||
setUpFunction: { setUp() },
|
||||
tearDownFunction: tearDown
|
||||
),
|
||||
BenchmarkInfo(
|
||||
name: "BaggagePassingBenchmarks.pass_async_small_nonconst_100_000",
|
||||
runFunction: { _ in
|
||||
var context = BaggageContext()
|
||||
context.k1 = "\(Int.random(in: 1 ... Int.max))"
|
||||
context.k2 = "\(Int.random(in: 1 ... Int.max))"
|
||||
context.k3 = "\(Int.random(in: 1 ... Int.max))"
|
||||
context.k4 = "\(Int.random(in: 1 ... Int.max))"
|
||||
pass_async(context: context, times: 100_000)
|
||||
},
|
||||
tags: [],
|
||||
setUpFunction: { setUp() },
|
||||
tearDownFunction: tearDown
|
||||
),
|
||||
|
||||
// ==== ------------------------------------------------------------------------------------------------------------
|
||||
// MARK: Passing & Mutating
|
||||
// Since the context is backed by a dictionary (and nothing else) we rely on its CoW semantics, those writes cause copies
|
||||
// whilst the previous benchmarks which are read-only do not cause copies of the underlying storage (dictionary).
|
||||
|
||||
BenchmarkInfo(
|
||||
name: "BaggagePassingBenchmarks.pass_mut_async_small_100_000 ",
|
||||
runFunction: { _ in
|
||||
var context = BaggageContext()
|
||||
context.k1 = "\(Int.random(in: 1 ... Int.max))"
|
||||
context.k2 = "\(Int.random(in: 1 ... Int.max))"
|
||||
context.k3 = "\(Int.random(in: 1 ... Int.max))"
|
||||
context.k4 = "\(Int.random(in: 1 ... Int.max))"
|
||||
pass_mut_async(context: context, times: 100_000)
|
||||
},
|
||||
tags: [],
|
||||
setUpFunction: { setUp() },
|
||||
tearDownFunction: tearDown
|
||||
),
|
||||
]
|
||||
|
||||
private func setUp() {
|
||||
// ...
|
||||
}
|
||||
|
||||
private func tearDown() {
|
||||
// ...
|
||||
}
|
||||
|
||||
@inline(never)
|
||||
func pass_async(context: BaggageContext, times remaining: Int) {
|
||||
let latch = CountDownLatch(from: 1)
|
||||
|
||||
func pass_async0(context: BaggageContext, times remaining: Int) {
|
||||
if remaining == 0 {
|
||||
latch.countDown()
|
||||
}
|
||||
DispatchQueue.global().async {
|
||||
pass_async0(context: context, times: remaining - 1)
|
||||
}
|
||||
}
|
||||
|
||||
pass_async0(context: context, times: remaining - 1)
|
||||
|
||||
latch.wait()
|
||||
}
|
||||
|
||||
@inline(never)
|
||||
func pass_mut_async(context: BaggageContext, times remaining: Int) {
|
||||
var context = context
|
||||
let latch = CountDownLatch(from: 1)
|
||||
|
||||
func pass_async0(context: BaggageContext, times remaining: Int) {
|
||||
if remaining == 0 {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
DispatchQueue.global().async {
|
||||
// mutate the context
|
||||
var context = context
|
||||
context.passCounter = remaining
|
||||
|
||||
pass_async0(context: context, times: remaining - 1)
|
||||
}
|
||||
}
|
||||
|
||||
context.passCounter = remaining
|
||||
pass_async0(context: context, times: remaining - 1)
|
||||
|
||||
latch.wait()
|
||||
}
|
||||
|
||||
// ==== ----------------------------------------------------------------------------------------------------------------
|
||||
// MARK: Baggage Keys
|
||||
|
||||
private enum TestPassCounterKey: BaggageContextKey {
|
||||
typealias Value = Int
|
||||
}
|
||||
|
||||
private enum TestK1: BaggageContextKey {
|
||||
typealias Value = String
|
||||
}
|
||||
|
||||
private enum TestK2: BaggageContextKey {
|
||||
typealias Value = String
|
||||
}
|
||||
|
||||
private enum TestK3: BaggageContextKey {
|
||||
typealias Value = String
|
||||
}
|
||||
|
||||
private enum TestK4: BaggageContextKey {
|
||||
typealias Value = String
|
||||
}
|
||||
|
||||
private enum TestKD1: BaggageContextKey {
|
||||
typealias Value = [String: String]
|
||||
}
|
||||
|
||||
extension BaggageContext {
|
||||
fileprivate var passCounter: TestPassCounterKey.Value {
|
||||
get { self[TestPassCounterKey.self] ?? 0 }
|
||||
set { self[TestPassCounterKey.self] = newValue }
|
||||
}
|
||||
|
||||
fileprivate var k1: TestK1.Value? {
|
||||
get { self[TestK1.self] }
|
||||
set { self[TestK1.self] = newValue }
|
||||
}
|
||||
|
||||
fileprivate var k2: TestK2.Value? {
|
||||
get { self[TestK2.self] }
|
||||
set { self[TestK2.self] = newValue }
|
||||
}
|
||||
|
||||
fileprivate var k3: TestK3.Value? {
|
||||
get { self[TestK3.self] }
|
||||
set { self[TestK3.self] = newValue }
|
||||
}
|
||||
|
||||
fileprivate var k4: TestK4.Value? {
|
||||
get { self[TestK4.self] }
|
||||
set { self[TestK4.self] = newValue }
|
||||
}
|
||||
|
||||
fileprivate var kd1: TestKD1.Value? {
|
||||
get { self[TestKD1.self] }
|
||||
set { self[TestKD1.self] = newValue }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,253 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift Baggage Context open source project
|
||||
//
|
||||
// Copyright (c) 2020 Moritz Lang and the Swift Baggage Context project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS)
|
||||
import Darwin
|
||||
#else
|
||||
import Glibc
|
||||
#endif
|
||||
|
||||
// ==== ----------------------------------------------------------------------------------------------------------------
|
||||
// MARK: CountDownLatch
|
||||
|
||||
internal class CountDownLatch {
|
||||
private var counter: Int
|
||||
private let condition: _Condition
|
||||
private let lock: _Mutex
|
||||
|
||||
init(from: Int) {
|
||||
self.counter = from
|
||||
self.condition = _Condition()
|
||||
self.lock = _Mutex()
|
||||
}
|
||||
|
||||
/// Returns previous value before the decrement was issued.
|
||||
func countDown() {
|
||||
self.lock.synchronized {
|
||||
self.counter -= 1
|
||||
|
||||
if self.counter == 0 {
|
||||
self.condition.signalAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var count: Int {
|
||||
self.lock.synchronized {
|
||||
self.counter
|
||||
}
|
||||
}
|
||||
|
||||
func wait() {
|
||||
self.lock.synchronized {
|
||||
while true {
|
||||
if self.counter == 0 {
|
||||
return // done
|
||||
}
|
||||
|
||||
self.condition.wait(lock)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension CountDownLatch: CustomStringConvertible {
|
||||
public var description: String {
|
||||
"CountDownLatch(remaining:\(self.count)"
|
||||
}
|
||||
}
|
||||
|
||||
// ==== ----------------------------------------------------------------------------------------------------------------
|
||||
// MARK: Condition
|
||||
|
||||
final class _Condition {
|
||||
@usableFromInline
|
||||
var condition: pthread_cond_t = pthread_cond_t()
|
||||
|
||||
public init() {
|
||||
let error = pthread_cond_init(&self.condition, nil)
|
||||
|
||||
switch error {
|
||||
case 0:
|
||||
return
|
||||
default:
|
||||
fatalError("Condition could not be created: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
pthread_cond_destroy(&condition)
|
||||
}
|
||||
|
||||
@inlinable
|
||||
public func wait(_ mutex: _Mutex) {
|
||||
let error = pthread_cond_wait(&self.condition, &mutex.mutex)
|
||||
|
||||
switch error {
|
||||
case 0:
|
||||
return
|
||||
case EPERM:
|
||||
fatalError("Wait failed, mutex is not owned by this thread")
|
||||
case EINVAL:
|
||||
fatalError("Wait failed, condition is not valid")
|
||||
default:
|
||||
fatalError("Wait failed with unspecified error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
// @inlinable
|
||||
// public func wait(_ mutex: _Mutex) -> Bool {
|
||||
// let error = withUnsafePointer(to: time) { p -> Int32 in
|
||||
// #if os(macOS) || os(iOS) || os(tvOS) || os(watchOS)
|
||||
// return pthread_cond_timedwait_relative_np(&condition, &mutex.mutex, p)
|
||||
// #else
|
||||
// return pthread_cond_timedwait(&condition, &mutex.mutex, p)
|
||||
// #endif
|
||||
// }
|
||||
//
|
||||
// switch error {
|
||||
// case 0:
|
||||
// return true
|
||||
// case ETIMEDOUT:
|
||||
// return false
|
||||
// case EPERM:
|
||||
// fatalError("Wait failed, mutex is not owned by this thread")
|
||||
// case EINVAL:
|
||||
// fatalError("Wait failed, condition is not valid")
|
||||
// default:
|
||||
// fatalError("Wait failed with unspecified error: \(error)")
|
||||
// }
|
||||
// }
|
||||
|
||||
@inlinable
|
||||
public func signal() {
|
||||
let error = pthread_cond_signal(&self.condition)
|
||||
|
||||
switch error {
|
||||
case 0:
|
||||
return
|
||||
case EINVAL:
|
||||
fatalError("Signal failed, condition is not valid")
|
||||
default:
|
||||
fatalError("Signal failed with unspecified error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
@inlinable
|
||||
public func signalAll() {
|
||||
let error = pthread_cond_broadcast(&self.condition)
|
||||
|
||||
switch error {
|
||||
case 0:
|
||||
return
|
||||
case EINVAL:
|
||||
fatalError("Signal failed, condition is not valid")
|
||||
default:
|
||||
fatalError("Signal failed with unspecified error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ==== ----------------------------------------------------------------------------------------------------------------
|
||||
// MARK: Mutex
|
||||
|
||||
final class _Mutex {
|
||||
@usableFromInline
|
||||
var mutex: pthread_mutex_t = pthread_mutex_t()
|
||||
|
||||
public init() {
|
||||
var attr: pthread_mutexattr_t = pthread_mutexattr_t()
|
||||
pthread_mutexattr_init(&attr)
|
||||
pthread_mutexattr_settype(&attr, Int32(PTHREAD_MUTEX_RECURSIVE))
|
||||
|
||||
let error = pthread_mutex_init(&self.mutex, &attr)
|
||||
pthread_mutexattr_destroy(&attr)
|
||||
|
||||
switch error {
|
||||
case 0:
|
||||
return
|
||||
default:
|
||||
fatalError("Could not create mutex: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
pthread_mutex_destroy(&mutex)
|
||||
}
|
||||
|
||||
@inlinable
|
||||
public func lock() {
|
||||
let error = pthread_mutex_lock(&self.mutex)
|
||||
|
||||
switch error {
|
||||
case 0:
|
||||
return
|
||||
case EDEADLK:
|
||||
fatalError("Mutex could not be acquired because it would have caused a deadlock")
|
||||
default:
|
||||
fatalError("Failed with unspecified error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
@inlinable
|
||||
public func unlock() {
|
||||
let error = pthread_mutex_unlock(&self.mutex)
|
||||
|
||||
switch error {
|
||||
case 0:
|
||||
return
|
||||
case EPERM:
|
||||
fatalError("Mutex could not be unlocked because it is not held by the current thread")
|
||||
default:
|
||||
fatalError("Unlock failed with unspecified error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
@inlinable
|
||||
public func tryLock() -> Bool {
|
||||
let error = pthread_mutex_trylock(&self.mutex)
|
||||
|
||||
switch error {
|
||||
case 0:
|
||||
return true
|
||||
case EBUSY:
|
||||
return false
|
||||
case EDEADLK:
|
||||
fatalError("Mutex could not be acquired because it would have caused a deadlock")
|
||||
default:
|
||||
fatalError("Failed with unspecified error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
@inlinable
|
||||
public func synchronized<A>(_ f: () -> A) -> A {
|
||||
self.lock()
|
||||
|
||||
defer {
|
||||
unlock()
|
||||
}
|
||||
|
||||
return f()
|
||||
}
|
||||
|
||||
@inlinable
|
||||
public func synchronized<A>(_ f: () throws -> A) throws -> A {
|
||||
self.lock()
|
||||
|
||||
defer {
|
||||
unlock()
|
||||
}
|
||||
|
||||
return try f()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift Baggage Context open source project
|
||||
//
|
||||
// Copyright (c) 2020 Moritz Lang and the Swift Baggage Context project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import SwiftBenchmarkTools
|
||||
|
||||
assert({
|
||||
print("===========================================================================")
|
||||
print("= !! YOU ARE RUNNING BENCHMARKS IN DEBUG MODE !! =")
|
||||
print("= When running on the command line, use: `swift run -c release` =")
|
||||
print("===========================================================================")
|
||||
return true
|
||||
}())
|
||||
|
||||
@inline(__always)
|
||||
private func registerBenchmark(_ bench: BenchmarkInfo) {
|
||||
registeredBenchmarks.append(bench)
|
||||
}
|
||||
|
||||
@inline(__always)
|
||||
private func registerBenchmark(_ benches: [BenchmarkInfo]) {
|
||||
benches.forEach(registerBenchmark)
|
||||
}
|
||||
|
||||
@inline(__always)
|
||||
private func registerBenchmark(_ name: String, _ function: @escaping (Int) -> Void, _ tags: [BenchmarkCategory]) {
|
||||
registerBenchmark(BenchmarkInfo(name: name, runFunction: function, tags: tags))
|
||||
}
|
||||
|
||||
registerBenchmark(BaggagePassingBenchmarks)
|
||||
|
||||
main()
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift Baggage Context open source project
|
||||
//
|
||||
// Copyright (c) 2020 Moritz Lang and the Swift Baggage Context project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
//===--- ArgParse.swift ---------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift.org open source project
|
||||
//
|
||||
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
|
||||
// Licensed under Apache License v2.0 with Runtime Library Exception
|
||||
//
|
||||
// See https://swift.org/LICENSE.txt for license information
|
||||
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import Foundation
|
||||
|
||||
enum ArgumentError: Error {
|
||||
case missingValue(String)
|
||||
case invalidType(value: String, type: String, argument: String?)
|
||||
case unsupportedArgument(String)
|
||||
}
|
||||
|
||||
extension ArgumentError: CustomStringConvertible {
|
||||
public var description: String {
|
||||
switch self {
|
||||
case .missingValue(let key):
|
||||
return "missing value for '\(key)'"
|
||||
case .invalidType(let value, let type, let argument):
|
||||
return (argument == nil)
|
||||
? "'\(value)' is not a valid '\(type)'"
|
||||
: "'\(value)' is not a valid '\(type)' for '\(argument!)'"
|
||||
case .unsupportedArgument(let argument):
|
||||
return "unsupported argument '\(argument)'"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Type-checked parsing of the argument value.
|
||||
///
|
||||
/// - Returns: Typed value of the argument converted using the `parse` function.
|
||||
///
|
||||
/// - Throws: `ArgumentError.invalidType` when the conversion fails.
|
||||
func checked<T>(
|
||||
_ parse: (String) throws -> T?,
|
||||
_ value: String,
|
||||
argument: String? = nil
|
||||
) throws -> T {
|
||||
if let t = try parse(value) { return t }
|
||||
var type = "\(T.self)"
|
||||
if type.starts(with: "Optional<") {
|
||||
let s = type.index(after: type.firstIndex(of: "<")!)
|
||||
let e = type.index(before: type.endIndex) // ">"
|
||||
type = String(type[s ..< e]) // strip Optional< >
|
||||
}
|
||||
throw ArgumentError.invalidType(
|
||||
value: value, type: type, argument: argument
|
||||
)
|
||||
}
|
||||
|
||||
/// Parser that converts the program's command line arguments to typed values
|
||||
/// according to the parser's configuration, storing them in the provided
|
||||
/// instance of a value-holding type.
|
||||
class ArgumentParser<U> {
|
||||
private var result: U
|
||||
private var validOptions: [String] {
|
||||
self.arguments.compactMap { $0.name }
|
||||
}
|
||||
|
||||
private var arguments: [Argument] = []
|
||||
private let programName: String = {
|
||||
// Strip full path from the program name.
|
||||
let r = CommandLine.arguments[0].reversed()
|
||||
let ss = r[r.startIndex ..< (r.firstIndex(of: "/") ?? r.endIndex)]
|
||||
return String(ss.reversed())
|
||||
}()
|
||||
|
||||
private var positionalArgs = [String]()
|
||||
private var optionalArgsMap = [String: String]()
|
||||
|
||||
/// Argument holds the name of the command line parameter, its help
|
||||
/// desciption and a rule that's applied to process it.
|
||||
///
|
||||
/// The the rule is typically a value processing closure used to convert it
|
||||
/// into given type and storing it in the parsing result.
|
||||
///
|
||||
/// See also: addArgument, parseArgument
|
||||
struct Argument {
|
||||
let name: String?
|
||||
let help: String?
|
||||
let apply: () throws -> Void
|
||||
}
|
||||
|
||||
/// ArgumentParser is initialized with an instance of a type that holds
|
||||
/// the results of the parsing of the individual command line arguments.
|
||||
init(into result: U) {
|
||||
self.result = result
|
||||
self.arguments += [
|
||||
Argument(
|
||||
name: "--help", help: "show this help message and exit",
|
||||
apply: self.printUsage
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
private func printUsage() {
|
||||
guard let _ = optionalArgsMap["--help"] else { return }
|
||||
let space = " "
|
||||
let maxLength = self.arguments.compactMap { $0.name?.count }.max()!
|
||||
let padded = { (s: String) in
|
||||
" \(s)\(String(repeating: space, count: maxLength - s.count)) "
|
||||
}
|
||||
let f: (String, String) -> String = {
|
||||
"\(padded($0))\($1)"
|
||||
.split(separator: "\n")
|
||||
.joined(separator: "\n" + padded(""))
|
||||
}
|
||||
let positional = f("TEST", "name or number of the benchmark to measure")
|
||||
let optional = self.arguments.filter { $0.name != nil }
|
||||
.map { f($0.name!, $0.help ?? "") }
|
||||
.joined(separator: "\n")
|
||||
print(
|
||||
"""
|
||||
usage: \(self.programName) [--argument=VALUE] [TEST [TEST ...]]
|
||||
positional arguments:
|
||||
\(positional)
|
||||
optional arguments:
|
||||
\(optional)
|
||||
""")
|
||||
exit(0)
|
||||
}
|
||||
|
||||
/// Parses the command line arguments, returning the result filled with
|
||||
/// specified argument values or report errors and exit the program if
|
||||
/// the parsing fails.
|
||||
public func parse() -> U {
|
||||
do {
|
||||
try self.parseArgs() // parse the argument syntax
|
||||
try self.arguments.forEach { try $0.apply() } // type-check and store values
|
||||
return self.result
|
||||
} catch let error as ArgumentError {
|
||||
fputs("error: \(error)\n", stderr)
|
||||
exit(1)
|
||||
} catch {
|
||||
fflush(stdout)
|
||||
fatalError("\(error)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Using CommandLine.arguments, parses the structure of optional and
|
||||
/// positional arguments of this program.
|
||||
///
|
||||
/// We assume that optional switch args are of the form:
|
||||
///
|
||||
/// --opt-name[=opt-value]
|
||||
/// -opt-name[=opt-value]
|
||||
///
|
||||
/// with `opt-name` and `opt-value` not containing any '=' signs. Any
|
||||
/// other option passed in is assumed to be a positional argument.
|
||||
///
|
||||
/// - Throws: `ArgumentError.unsupportedArgument` on failure to parse
|
||||
/// the supported argument syntax.
|
||||
private func parseArgs() throws {
|
||||
// For each argument we are passed...
|
||||
for arg in CommandLine.arguments[1 ..< CommandLine.arguments.count] {
|
||||
// If the argument doesn't match the optional argument pattern. Add
|
||||
// it to the positional argument list and continue...
|
||||
if !arg.starts(with: "-") {
|
||||
self.positionalArgs.append(arg)
|
||||
continue
|
||||
}
|
||||
// Attempt to split it into two components separated by an equals sign.
|
||||
let components = arg.split(separator: "=")
|
||||
let optionName = String(components[0])
|
||||
guard self.validOptions.contains(optionName) else {
|
||||
throw ArgumentError.unsupportedArgument(arg)
|
||||
}
|
||||
var optionVal: String
|
||||
switch components.count {
|
||||
case 1: optionVal = ""
|
||||
case 2: optionVal = String(components[1])
|
||||
default:
|
||||
// If we do not have two components at this point, we can not have
|
||||
// an option switch. This is an invalid argument. Bail!
|
||||
throw ArgumentError.unsupportedArgument(arg)
|
||||
}
|
||||
self.optionalArgsMap[optionName] = optionVal
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a rule for parsing the specified argument.
|
||||
///
|
||||
/// Stores the type-erased invocation of the `parseArgument` in `Argument`.
|
||||
///
|
||||
/// Parameters:
|
||||
/// - name: Name of the command line argument. E.g.: `--opt-arg`.
|
||||
/// `nil` denotes positional arguments.
|
||||
/// - property: Property on the `result`, to store the value into.
|
||||
/// - defaultValue: Value used when the command line argument doesn't
|
||||
/// provide one.
|
||||
/// - help: Argument's description used when printing usage with `--help`.
|
||||
/// - parser: Function that converts the argument value to given type `T`.
|
||||
public func addArgument<T>(
|
||||
_ name: String?,
|
||||
_ property: WritableKeyPath<U, T>,
|
||||
defaultValue: T? = nil,
|
||||
help: String? = nil,
|
||||
parser: @escaping (String) throws -> T? = { _ in nil }
|
||||
) {
|
||||
self.arguments.append(
|
||||
Argument(name: name, help: help)
|
||||
{ try self.parseArgument(name, property, defaultValue, parser) }
|
||||
)
|
||||
}
|
||||
|
||||
/// Process the specified command line argument.
|
||||
///
|
||||
/// For optional arguments that have a value we attempt to convert it into
|
||||
/// given type using the supplied parser, performing the type-checking with
|
||||
/// the `checked` function.
|
||||
/// If the value is empty the `defaultValue` is used instead.
|
||||
/// The typed value is finally stored in the `result` into the specified
|
||||
/// `property`.
|
||||
///
|
||||
/// For the optional positional arguments, the [String] is simply assigned
|
||||
/// to the specified property without any conversion.
|
||||
///
|
||||
/// See `addArgument` for detailed parameter descriptions.
|
||||
private func parseArgument<T>(
|
||||
_ name: String?,
|
||||
_ property: WritableKeyPath<U, T>,
|
||||
_ defaultValue: T?,
|
||||
_ parse: (String) throws -> T?
|
||||
) throws {
|
||||
if let name = name, let value = optionalArgsMap[name] {
|
||||
guard !value.isEmpty || defaultValue != nil
|
||||
else { throw ArgumentError.missingValue(name) }
|
||||
|
||||
self.result[keyPath: property] = value.isEmpty
|
||||
? defaultValue!
|
||||
: try checked(parse, value, argument: name)
|
||||
} else if name == nil {
|
||||
self.result[keyPath: property] = self.positionalArgs as! T
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift Baggage Context open source project
|
||||
//
|
||||
// Copyright (c) 2020 Moritz Lang and the Swift Baggage Context project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Based on: https://github.com/apple/swift/tree/cf53143a47278c2a465409a67376642515956777/benchmark/utils
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
public enum BenchmarkCategory: String {
|
||||
// Most benchmarks are assumed to be "stable" and will be regularly tracked at
|
||||
// each commit. A handful may be marked unstable if continually tracking them is
|
||||
// counterproductive.
|
||||
case unstable
|
||||
|
||||
// Explicit skip marker
|
||||
case skip
|
||||
}
|
||||
|
|
@ -0,0 +1,230 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift Baggage Context open source project
|
||||
//
|
||||
// Copyright (c) 2020 Moritz Lang and the Swift Baggage Context project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Based on: https://github.com/apple/swift/tree/cf53143a47278c2a465409a67376642515956777/benchmark/utils
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#if os(Linux)
|
||||
import Glibc
|
||||
#else
|
||||
import Darwin
|
||||
#endif
|
||||
|
||||
extension BenchmarkCategory: CustomStringConvertible {
|
||||
public var description: String {
|
||||
self.rawValue
|
||||
}
|
||||
}
|
||||
|
||||
extension BenchmarkCategory: Comparable {
|
||||
public static func < (lhs: BenchmarkCategory, rhs: BenchmarkCategory) -> Bool {
|
||||
lhs.rawValue < rhs.rawValue
|
||||
}
|
||||
}
|
||||
|
||||
public struct BenchmarkPlatformSet: OptionSet {
|
||||
public let rawValue: Int
|
||||
|
||||
public init(rawValue: Int) {
|
||||
self.rawValue = rawValue
|
||||
}
|
||||
|
||||
public static let darwin = BenchmarkPlatformSet(rawValue: 1 << 0)
|
||||
public static let linux = BenchmarkPlatformSet(rawValue: 1 << 1)
|
||||
|
||||
public static var currentPlatform: BenchmarkPlatformSet {
|
||||
#if os(Linux)
|
||||
return .linux
|
||||
#else
|
||||
return .darwin
|
||||
#endif
|
||||
}
|
||||
|
||||
public static var allPlatforms: BenchmarkPlatformSet {
|
||||
[.darwin, .linux]
|
||||
}
|
||||
}
|
||||
|
||||
public struct BenchmarkInfo {
|
||||
/// The name of the benchmark that should be displayed by the harness.
|
||||
public var name: String
|
||||
|
||||
/// Shadow static variable for runFunction.
|
||||
private var _runFunction: (Int) -> Void
|
||||
|
||||
/// A function that invokes the specific benchmark routine.
|
||||
public var runFunction: ((Int) -> Void)? {
|
||||
if !self.shouldRun {
|
||||
return nil
|
||||
}
|
||||
return self._runFunction
|
||||
}
|
||||
|
||||
/// A set of category tags that describe this benchmark. This is used by the
|
||||
/// harness to allow for easy slicing of the set of benchmarks along tag
|
||||
/// boundaries, e.x.: run all string benchmarks or ref count benchmarks, etc.
|
||||
public var tags: Set<BenchmarkCategory>
|
||||
|
||||
/// The platforms that this benchmark supports. This is an OptionSet.
|
||||
private var unsupportedPlatforms: BenchmarkPlatformSet
|
||||
|
||||
/// Shadow variable for setUpFunction.
|
||||
private var _setUpFunction: (() -> Void)?
|
||||
|
||||
/// An optional function that if non-null is run before benchmark samples
|
||||
/// are timed.
|
||||
public var setUpFunction: (() -> Void)? {
|
||||
if !self.shouldRun {
|
||||
return nil
|
||||
}
|
||||
return self._setUpFunction
|
||||
}
|
||||
|
||||
/// Shadow static variable for computed property tearDownFunction.
|
||||
private var _tearDownFunction: (() -> Void)?
|
||||
|
||||
/// An optional function that if non-null is run after samples are taken.
|
||||
public var tearDownFunction: (() -> Void)? {
|
||||
if !self.shouldRun {
|
||||
return nil
|
||||
}
|
||||
return self._tearDownFunction
|
||||
}
|
||||
|
||||
public var legacyFactor: Int?
|
||||
|
||||
public init(
|
||||
name: String, runFunction: @escaping (Int) -> Void, tags: [BenchmarkCategory],
|
||||
setUpFunction: (() -> Void)? = nil,
|
||||
tearDownFunction: (() -> Void)? = nil,
|
||||
unsupportedPlatforms: BenchmarkPlatformSet = [],
|
||||
legacyFactor: Int? = nil
|
||||
) {
|
||||
self.name = name
|
||||
self._runFunction = runFunction
|
||||
self.tags = Set(tags)
|
||||
self._setUpFunction = setUpFunction
|
||||
self._tearDownFunction = tearDownFunction
|
||||
self.unsupportedPlatforms = unsupportedPlatforms
|
||||
self.legacyFactor = legacyFactor
|
||||
}
|
||||
|
||||
/// Returns true if this benchmark should be run on the current platform.
|
||||
var shouldRun: Bool {
|
||||
!self.unsupportedPlatforms.contains(.currentPlatform)
|
||||
}
|
||||
}
|
||||
|
||||
extension BenchmarkInfo: Comparable {
|
||||
public static func < (lhs: BenchmarkInfo, rhs: BenchmarkInfo) -> Bool {
|
||||
lhs.name < rhs.name
|
||||
}
|
||||
|
||||
public static func == (lhs: BenchmarkInfo, rhs: BenchmarkInfo) -> Bool {
|
||||
lhs.name == rhs.name
|
||||
}
|
||||
}
|
||||
|
||||
extension BenchmarkInfo: Hashable {
|
||||
public func hash(into hasher: inout Hasher) {
|
||||
hasher.combine(self.name)
|
||||
}
|
||||
}
|
||||
|
||||
// Linear function shift register.
|
||||
//
|
||||
// This is just to drive benchmarks. I don't make any claim about its
|
||||
// strength. According to Wikipedia, it has the maximal period for a
|
||||
// 32-bit register.
|
||||
struct LFSR {
|
||||
// Set the register to some seed that I pulled out of a hat.
|
||||
var lfsr: UInt32 = 0xB789_78E7
|
||||
|
||||
mutating func shift() {
|
||||
self.lfsr = (self.lfsr >> 1) ^ (UInt32(bitPattern: -Int32(self.lfsr & 1)) & 0xD000_0001)
|
||||
}
|
||||
|
||||
mutating func randInt() -> Int64 {
|
||||
var result: UInt32 = 0
|
||||
for _ in 0 ..< 32 {
|
||||
result = (result << 1) | (self.lfsr & 1)
|
||||
self.shift()
|
||||
}
|
||||
return Int64(bitPattern: UInt64(result))
|
||||
}
|
||||
}
|
||||
|
||||
var lfsrRandomGenerator = LFSR()
|
||||
|
||||
// Start the generator from the beginning
|
||||
public func SRand() {
|
||||
lfsrRandomGenerator = LFSR()
|
||||
}
|
||||
|
||||
public func Random() -> Int64 {
|
||||
lfsrRandomGenerator.randInt()
|
||||
}
|
||||
|
||||
@inlinable // FIXME(inline-always)
|
||||
@inline(__always)
|
||||
public func CheckResults(
|
||||
_ resultsMatch: Bool,
|
||||
file: StaticString = #file,
|
||||
function: StaticString = #function,
|
||||
line: Int = #line
|
||||
) {
|
||||
guard _fastPath(resultsMatch) else {
|
||||
print("Incorrect result in \(function), \(file):\(line)")
|
||||
abort()
|
||||
}
|
||||
}
|
||||
|
||||
public func False() -> Bool { false }
|
||||
|
||||
/// This is a dummy protocol to test the speed of our protocol dispatch.
|
||||
public protocol SomeProtocol { func getValue() -> Int }
|
||||
struct MyStruct: SomeProtocol {
|
||||
init() {}
|
||||
func getValue() -> Int { 1 }
|
||||
}
|
||||
|
||||
public func someProtocolFactory() -> SomeProtocol { MyStruct() }
|
||||
|
||||
// Just consume the argument.
|
||||
// It's important that this function is in another module than the tests
|
||||
// which are using it.
|
||||
@inline(never)
|
||||
public func blackHole<T>(_: T) {}
|
||||
|
||||
// Return the passed argument without letting the optimizer know that.
|
||||
@inline(never)
|
||||
public func identity<T>(_ x: T) -> T {
|
||||
x
|
||||
}
|
||||
|
||||
// Return the passed argument without letting the optimizer know that.
|
||||
// It's important that this function is in another module than the tests
|
||||
// which are using it.
|
||||
@inline(never)
|
||||
public func getInt(_ x: Int) -> Int { x }
|
||||
|
||||
// The same for String.
|
||||
@inline(never)
|
||||
public func getString(_ s: String) -> String { s }
|
||||
|
||||
// The same for Substring.
|
||||
@inline(never)
|
||||
public func getSubstring(_ s: Substring) -> Substring { s }
|
||||
|
|
@ -0,0 +1,722 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift Baggage Context open source project
|
||||
//
|
||||
// Copyright (c) 2020 Moritz Lang and the Swift Baggage Context project authors
|
||||
// Licensed under Apache License v2.0
|
||||
//
|
||||
// See LICENSE.txt for license information
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
//===--- DriverUtils.swift ------------------------------------------------===//
|
||||
//
|
||||
// This source file is part of the Swift.org open source project
|
||||
//
|
||||
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
|
||||
// Licensed under Apache License v2.0 with Runtime Library Exception
|
||||
//
|
||||
// See https://swift.org/LICENSE.txt for license information
|
||||
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#if os(Linux)
|
||||
import Glibc
|
||||
#else
|
||||
import Darwin
|
||||
// import LibProc
|
||||
#endif
|
||||
|
||||
public struct BenchResults {
|
||||
typealias T = Int
|
||||
private let samples: [T]
|
||||
let maxRSS: Int?
|
||||
let stats: Stats
|
||||
|
||||
init(_ samples: [T], maxRSS: Int?) {
|
||||
self.samples = samples.sorted()
|
||||
self.maxRSS = maxRSS
|
||||
self.stats = self.samples.reduce(into: Stats(), Stats.collect)
|
||||
}
|
||||
|
||||
/// Return measured value for given `quantile`.
|
||||
///
|
||||
/// Equivalent to quantile estimate type R-1, SAS-3. See:
|
||||
/// https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample
|
||||
subscript(_ quantile: Double) -> T {
|
||||
let index = Swift.max(
|
||||
0,
|
||||
Int((Double(self.samples.count) * quantile).rounded(.up)) - 1
|
||||
)
|
||||
return self.samples[index]
|
||||
}
|
||||
|
||||
var sampleCount: T { self.samples.count }
|
||||
var min: T { self.samples.first! }
|
||||
var max: T { self.samples.last! }
|
||||
var mean: T { Int(self.stats.mean.rounded()) }
|
||||
var sd: T { Int(self.stats.standardDeviation.rounded()) }
|
||||
var median: T { self[0.5] }
|
||||
}
|
||||
|
||||
public var registeredBenchmarks: [BenchmarkInfo] = []
|
||||
|
||||
enum TestAction {
|
||||
case run
|
||||
case listTests
|
||||
}
|
||||
|
||||
struct TestConfig {
|
||||
/// The delimiter to use when printing output.
|
||||
let delim: String
|
||||
|
||||
/// Duration of the test measurement in seconds.
|
||||
///
|
||||
/// Used to compute the number of iterations, if no fixed amount is specified.
|
||||
/// This is useful when one wishes for a test to run for a
|
||||
/// longer amount of time to perform performance analysis on the test in
|
||||
/// instruments.
|
||||
let sampleTime: Double
|
||||
|
||||
/// Number of iterations averaged in the sample.
|
||||
/// When not specified, we'll compute the number of iterations to be averaged
|
||||
/// in the sample from the actual runtime and the desired `sampleTime`.
|
||||
let numIters: Int?
|
||||
|
||||
/// The number of samples we should take of each test.
|
||||
let numSamples: Int?
|
||||
|
||||
/// Quantiles to report in results.
|
||||
let quantile: Int?
|
||||
|
||||
/// Time unit in which to report results (nanos, micros, millis) (default: nanoseconds)
|
||||
let timeUnit: TimeUnit
|
||||
|
||||
/// Report quantiles with delta encoding.
|
||||
let delta: Bool
|
||||
|
||||
/// Is verbose output enabled?
|
||||
let verbose: Bool
|
||||
|
||||
// Should we log the test's memory usage?
|
||||
let logMemory: Bool
|
||||
|
||||
/// After we run the tests, should the harness sleep to allow for utilities
|
||||
/// like leaks that require a PID to run on the test harness.
|
||||
let afterRunSleep: UInt32?
|
||||
|
||||
/// The list of tests to run.
|
||||
let tests: [(index: String, info: BenchmarkInfo)]
|
||||
|
||||
let action: TestAction
|
||||
|
||||
init(_ registeredBenchmarks: [BenchmarkInfo]) {
|
||||
struct PartialTestConfig {
|
||||
var delim: String?
|
||||
var tags, skipTags: Set<BenchmarkCategory>?
|
||||
var numSamples: UInt?
|
||||
var numIters: UInt?
|
||||
var quantile: UInt?
|
||||
var timeUnit: String?
|
||||
var delta: Bool?
|
||||
var afterRunSleep: UInt32?
|
||||
var sampleTime: Double?
|
||||
var verbose: Bool?
|
||||
var logMemory: Bool?
|
||||
var action: TestAction?
|
||||
var tests: [String]?
|
||||
}
|
||||
|
||||
// Custom value type parsers
|
||||
func tags(tags: String) throws -> Set<BenchmarkCategory> {
|
||||
// We support specifying multiple tags by splitting on comma, i.e.:
|
||||
// --tags=Array,Dictionary
|
||||
// --skip-tags=Array,Set,unstable,skip
|
||||
Set(
|
||||
try tags.split(separator: ",").map(String.init).map {
|
||||
try checked({ BenchmarkCategory(rawValue: $0) }, $0)
|
||||
}
|
||||
)
|
||||
}
|
||||
func finiteDouble(value: String) -> Double? {
|
||||
Double(value).flatMap { $0.isFinite ? $0 : nil }
|
||||
}
|
||||
|
||||
// Configure the command line argument parser
|
||||
let p = ArgumentParser(into: PartialTestConfig())
|
||||
p.addArgument(
|
||||
"--num-samples", \.numSamples,
|
||||
help: "number of samples to take per benchmark;\n" +
|
||||
"default: 1 or auto-scaled to measure for\n" +
|
||||
"`sample-time` if num-iters is also specified\n",
|
||||
parser: { UInt($0) }
|
||||
)
|
||||
p.addArgument(
|
||||
"--num-iters", \.numIters,
|
||||
help: "number of iterations averaged in the sample;\n" +
|
||||
"default: auto-scaled to measure for `sample-time`",
|
||||
parser: { UInt($0) }
|
||||
)
|
||||
p.addArgument(
|
||||
"--quantile", \.quantile,
|
||||
help: "report quantiles instead of normal dist. stats;\n" +
|
||||
"use 4 to get a five-number summary with quartiles,\n" +
|
||||
"10 (deciles), 20 (ventiles), 100 (percentiles), etc.",
|
||||
parser: { UInt($0) }
|
||||
)
|
||||
p.addArgument(
|
||||
"--time-unit", \.timeUnit,
|
||||
help: "time unit to be used for reported measurements;\n" +
|
||||
"supported values: ns, us, ms; default: ns",
|
||||
parser: { $0 }
|
||||
)
|
||||
p.addArgument(
|
||||
"--delta", \.delta, defaultValue: true,
|
||||
help: "report quantiles with delta encoding"
|
||||
)
|
||||
p.addArgument(
|
||||
"--sample-time", \.sampleTime,
|
||||
help: "duration of test measurement in seconds\ndefault: 1",
|
||||
parser: finiteDouble
|
||||
)
|
||||
p.addArgument(
|
||||
"--verbose", \.verbose, defaultValue: true,
|
||||
help: "increase output verbosity"
|
||||
)
|
||||
p.addArgument(
|
||||
"--memory", \.logMemory, defaultValue: true,
|
||||
help: "log the change in maximum resident set size (MAX_RSS)"
|
||||
)
|
||||
p.addArgument(
|
||||
"--delim", \.delim,
|
||||
help: "value delimiter used for log output; default: ,",
|
||||
parser: { $0 }
|
||||
)
|
||||
p.addArgument(
|
||||
"--tags", \PartialTestConfig.tags,
|
||||
help: "run tests matching all the specified categories",
|
||||
parser: tags
|
||||
)
|
||||
p.addArgument(
|
||||
"--skip-tags", \PartialTestConfig.skipTags, defaultValue: [],
|
||||
help: "don't run tests matching any of the specified\n" +
|
||||
"categories; default: unstable,skip",
|
||||
parser: tags
|
||||
)
|
||||
p.addArgument(
|
||||
"--sleep", \.afterRunSleep,
|
||||
help: "number of seconds to sleep after benchmarking",
|
||||
parser: { UInt32($0) }
|
||||
)
|
||||
p.addArgument(
|
||||
"--list", \.action, defaultValue: .listTests,
|
||||
help: "don't run the tests, just log the list of test \n" +
|
||||
"numbers, names and tags (respects specified filters)"
|
||||
)
|
||||
p.addArgument(nil, \.tests) // positional arguments
|
||||
let c = p.parse()
|
||||
|
||||
// Configure from the command line arguments, filling in the defaults.
|
||||
self.delim = c.delim ?? ","
|
||||
self.sampleTime = c.sampleTime ?? 1.0
|
||||
self.numIters = c.numIters.map { Int($0) }
|
||||
self.numSamples = c.numSamples.map { Int($0) }
|
||||
self.quantile = c.quantile.map { Int($0) }
|
||||
self.timeUnit = c.timeUnit.map { TimeUnit($0) } ?? TimeUnit.nanoseconds
|
||||
self.delta = c.delta ?? false
|
||||
self.verbose = c.verbose ?? false
|
||||
self.logMemory = c.logMemory ?? false
|
||||
self.afterRunSleep = c.afterRunSleep
|
||||
self.action = c.action ?? .run
|
||||
self.tests = TestConfig.filterTests(
|
||||
registeredBenchmarks,
|
||||
specifiedTests: Set(c.tests ?? []),
|
||||
tags: c.tags ?? [],
|
||||
skipTags: c.skipTags ?? [.unstable, .skip]
|
||||
)
|
||||
|
||||
if self.logMemory, self.tests.count > 1 {
|
||||
print(
|
||||
"""
|
||||
warning: The memory usage of a test, reported as the change in MAX_RSS,
|
||||
is based on measuring the peak memory used by the whole process.
|
||||
These results are meaningful only when running a single test,
|
||||
not in the batch mode!
|
||||
""")
|
||||
}
|
||||
|
||||
// We always prepare the configuration string and call the print to have
|
||||
// the same memory usage baseline between verbose and normal mode.
|
||||
let testList = self.tests.map { $0.1.name }.joined(separator: ", ")
|
||||
let configuration = """
|
||||
--- CONFIG ---
|
||||
NumSamples: \(numSamples ?? 0)
|
||||
Verbose: \(verbose)
|
||||
LogMemory: \(logMemory)
|
||||
SampleTime: \(sampleTime)
|
||||
NumIters: \(numIters ?? 0)
|
||||
Quantile: \(quantile ?? 0)
|
||||
TimeUnit: \(timeUnit)
|
||||
Delimiter: \(String(reflecting: delim))
|
||||
Tests Filter: \(c.tests ?? [])
|
||||
Tests to run: \(testList)
|
||||
--- DATA ---\n
|
||||
"""
|
||||
print(self.verbose ? configuration : "", terminator: "")
|
||||
}
|
||||
|
||||
/// Returns the list of tests to run.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - registeredBenchmarks: List of all performance tests to be filtered.
|
||||
/// - specifiedTests: List of explicitly specified tests to run. These can
|
||||
/// be specified either by a test name or a test number.
|
||||
/// - tags: Run tests tagged with all of these categories.
|
||||
/// - skipTags: Don't run tests tagged with any of these categories.
|
||||
/// - Returns: An array of test number and benchmark info tuples satisfying
|
||||
/// specified filtering conditions.
|
||||
static func filterTests(
|
||||
_ registeredBenchmarks: [BenchmarkInfo],
|
||||
specifiedTests: Set<String>,
|
||||
tags: Set<BenchmarkCategory>,
|
||||
skipTags: Set<BenchmarkCategory>
|
||||
) -> [(index: String, info: BenchmarkInfo)] {
|
||||
let allTests = registeredBenchmarks.sorted()
|
||||
let indices = Dictionary(
|
||||
uniqueKeysWithValues:
|
||||
zip(
|
||||
allTests.map { $0.name },
|
||||
(1...).lazy.map { String($0) }
|
||||
)
|
||||
)
|
||||
|
||||
func byTags(b: BenchmarkInfo) -> Bool {
|
||||
b.tags.isSuperset(of: tags) &&
|
||||
b.tags.isDisjoint(with: skipTags)
|
||||
}
|
||||
func byNamesOrIndices(b: BenchmarkInfo) -> Bool {
|
||||
specifiedTests.contains(b.name) ||
|
||||
specifiedTests.contains(indices[b.name]!)
|
||||
} // !! "`allTests` have been assigned an index"
|
||||
return allTests
|
||||
.filter(specifiedTests.isEmpty ? byTags : byNamesOrIndices)
|
||||
.map { (index: indices[$0.name]!, info: $0) }
|
||||
}
|
||||
}
|
||||
|
||||
struct Stats {
|
||||
var n: Int = 0
|
||||
var S: Double = 0.0
|
||||
var mean: Double = 0.0
|
||||
var variance: Double { self.n < 2 ? 0.0 : self.S / Double(self.n - 1) }
|
||||
var standardDeviation: Double { self.variance.squareRoot() }
|
||||
|
||||
static func collect(_ s: inout Stats, _ x: Int) {
|
||||
Stats.runningMeanVariance(&s, Double(x))
|
||||
}
|
||||
|
||||
/// Compute running mean and variance using B. P. Welford's method.
|
||||
///
|
||||
/// See Knuth TAOCP vol 2, 3rd edition, page 232, or
|
||||
/// https://www.johndcook.com/blog/standard_deviation/
|
||||
static func runningMeanVariance(_ s: inout Stats, _ x: Double) {
|
||||
let n = s.n + 1
|
||||
let (k, M_, S_) = (Double(n), s.mean, s.S)
|
||||
let M = M_ + (x - M_) / k
|
||||
let S = S_ + (x - M_) * (x - M)
|
||||
(s.n, s.mean, s.S) = (n, M, S)
|
||||
}
|
||||
}
|
||||
|
||||
#if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER
|
||||
|
||||
@_silgen_name("_swift_leaks_startTrackingObjects")
|
||||
func startTrackingObjects(_: UnsafePointer<CChar>) -> Void
|
||||
@_silgen_name("_swift_leaks_stopTrackingObjects")
|
||||
func stopTrackingObjects(_: UnsafePointer<CChar>) -> Int
|
||||
|
||||
#endif
|
||||
|
||||
public final class Timer {
|
||||
#if os(Linux)
|
||||
public typealias TimeT = timespec
|
||||
|
||||
public init() {}
|
||||
|
||||
public func getTime() -> TimeT {
|
||||
var ts = timespec(tv_sec: 0, tv_nsec: 0)
|
||||
clock_gettime(CLOCK_REALTIME, &ts)
|
||||
return ts
|
||||
}
|
||||
|
||||
public func getTimeAsInt() -> UInt64 {
|
||||
UInt64(getTime().tv_nsec)
|
||||
}
|
||||
|
||||
public func diffTimeInNanoSeconds(from start: TimeT, to end: TimeT) -> UInt64 {
|
||||
let oneSecond = 1_000_000_000 // ns
|
||||
var elapsed = timespec(tv_sec: 0, tv_nsec: 0)
|
||||
if end.tv_nsec - start.tv_nsec < 0 {
|
||||
elapsed.tv_sec = end.tv_sec - start.tv_sec - 1
|
||||
elapsed.tv_nsec = end.tv_nsec - start.tv_nsec + oneSecond
|
||||
} else {
|
||||
elapsed.tv_sec = end.tv_sec - start.tv_sec
|
||||
elapsed.tv_nsec = end.tv_nsec - start.tv_nsec
|
||||
}
|
||||
return UInt64(elapsed.tv_sec) * UInt64(oneSecond) + UInt64(elapsed.tv_nsec)
|
||||
}
|
||||
|
||||
#else
|
||||
public typealias TimeT = UInt64
|
||||
var info = mach_timebase_info_data_t(numer: 0, denom: 0)
|
||||
|
||||
public init() {
|
||||
mach_timebase_info(&info)
|
||||
}
|
||||
|
||||
public func getTime() -> TimeT {
|
||||
mach_absolute_time()
|
||||
}
|
||||
|
||||
public func getTimeAsInt() -> UInt64 {
|
||||
UInt64(getTime())
|
||||
}
|
||||
|
||||
public func diffTimeInNanoSeconds(from start: TimeT, to end: TimeT) -> UInt64 {
|
||||
let elapsed = end - start
|
||||
return elapsed * UInt64(info.numer) / UInt64(info.denom)
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
extension UInt64 {
|
||||
public var nanoseconds: Int { Int(self) }
|
||||
public var microseconds: Int { Int(self / 1000) }
|
||||
public var milliseconds: Int { Int(self / 1000 / 1000) }
|
||||
public var seconds: Int { Int(self / 1000 / 1000 / 1000) }
|
||||
}
|
||||
|
||||
enum TimeUnit: String {
|
||||
case nanoseconds = "ns"
|
||||
case microseconds = "μs"
|
||||
case milliseconds = "ms"
|
||||
case seconds = "s"
|
||||
|
||||
init(_ from: String) {
|
||||
switch from {
|
||||
case "ns": self = .nanoseconds
|
||||
case "us", "μs": self = .microseconds
|
||||
case "ms": self = .milliseconds
|
||||
case "s": self = .seconds
|
||||
default: fatalError("Only the following time units are supported: ns, us, ms, s")
|
||||
}
|
||||
}
|
||||
|
||||
static var `default` = TimeUnit.nanoseconds
|
||||
}
|
||||
|
||||
extension TimeUnit: CustomStringConvertible {
|
||||
public var description: String {
|
||||
self.rawValue
|
||||
}
|
||||
}
|
||||
|
||||
/// Performance test runner that measures benchmarks and reports the results.
|
||||
final class TestRunner {
|
||||
let c: TestConfig
|
||||
let timer = Timer()
|
||||
var start, end, lastYield: Timer.TimeT
|
||||
let baseline = TestRunner.getResourceUtilization()
|
||||
let schedulerQuantum = UInt64(10_000_000) // nanoseconds (== 10ms, macos)
|
||||
init(_ config: TestConfig) {
|
||||
self.c = config
|
||||
let now = timer.getTime()
|
||||
(start, end, lastYield) = (now, now, now)
|
||||
}
|
||||
|
||||
/// Offer to yield CPU to other processes and return current time on resume.
|
||||
func yield() -> Timer.TimeT {
|
||||
sched_yield()
|
||||
return timer.getTime()
|
||||
}
|
||||
|
||||
#if os(Linux)
|
||||
private static func getExecutedInstructions() -> UInt64 {
|
||||
// FIXME: there is a Linux PMC API you can use to get this, but it's
|
||||
// not quite so straightforward.
|
||||
0
|
||||
}
|
||||
|
||||
#else
|
||||
private static func getExecutedInstructions() -> UInt64 {
|
||||
// if #available(OSX 10.9, iOS 7.0, *) {
|
||||
// var u = rusage_info_v4()
|
||||
// let p = UnsafeMutablePointer(&u)
|
||||
// p.withMemoryRebound(to: Optional<rusage_info_t>.self, capacity: 1) { up in
|
||||
// let _ = proc_pid_rusage(getpid(), RUSAGE_INFO_V4, up)
|
||||
// }
|
||||
// return u.ri_instructions
|
||||
// } else {
|
||||
0
|
||||
// }
|
||||
}
|
||||
#endif
|
||||
|
||||
private static func getResourceUtilization() -> rusage {
|
||||
#if canImport(Darwin)
|
||||
let rusageSelf = RUSAGE_SELF
|
||||
#else
|
||||
let rusageSelf = RUSAGE_SELF.rawValue
|
||||
#endif
|
||||
var u = rusage(); getrusage(rusageSelf, &u); return u
|
||||
}
|
||||
|
||||
/// Returns maximum resident set size (MAX_RSS) delta in bytes.
|
||||
///
|
||||
/// This method of estimating memory usage is valid only for executing single
|
||||
/// benchmark. That's why we don't worry about reseting the `baseline` in
|
||||
/// `resetMeasurements`.
|
||||
///
|
||||
// FIXME: This current implementation doesn't work on Linux. It is disabled
|
||||
/// permanently to avoid linker errors. Feel free to fix.
|
||||
func measureMemoryUsage() -> Int? {
|
||||
#if os(Linux)
|
||||
return nil
|
||||
#else
|
||||
guard c.logMemory else { return nil }
|
||||
let current = TestRunner.getResourceUtilization()
|
||||
let maxRSS = current.ru_maxrss - baseline.ru_maxrss
|
||||
#if canImport(Darwin)
|
||||
let pageSize = _SC_PAGESIZE
|
||||
#else
|
||||
let pageSize = Int32(_SC_PAGESIZE)
|
||||
#endif
|
||||
let pages = { maxRSS / sysconf(pageSize) }
|
||||
func deltaEquation(_ stat: KeyPath<rusage, Int>) -> String {
|
||||
let b = baseline[keyPath: stat], c = current[keyPath: stat]
|
||||
return "\(c) - \(b) = \(c - b)"
|
||||
}
|
||||
logVerbose(
|
||||
"""
|
||||
MAX_RSS \(deltaEquation(\rusage.ru_maxrss)) (\(pages()) pages)
|
||||
ICS \(deltaEquation(\rusage.ru_nivcsw))
|
||||
VCS \(deltaEquation(\rusage.ru_nvcsw))
|
||||
""")
|
||||
return maxRSS
|
||||
#endif
|
||||
}
|
||||
|
||||
private func startMeasurement() {
|
||||
let spent = timer.diffTimeInNanoSeconds(from: lastYield, to: end)
|
||||
let nextSampleEstimate = UInt64(Double(lastSampleTime) * 1.5)
|
||||
|
||||
if spent + nextSampleEstimate < schedulerQuantum {
|
||||
start = timer.getTime()
|
||||
} else {
|
||||
logVerbose(" Yielding after ~\(spent.nanoseconds) ns")
|
||||
let now = yield()
|
||||
(start, lastYield) = (now, now)
|
||||
}
|
||||
}
|
||||
|
||||
private func stopMeasurement() {
|
||||
end = timer.getTime()
|
||||
}
|
||||
|
||||
private func resetMeasurements() {
|
||||
let now = yield()
|
||||
(start, end, lastYield) = (now, now, now)
|
||||
}
|
||||
|
||||
/// Time in nanoseconds spent running the last function
|
||||
var lastSampleTime: UInt64 {
|
||||
timer.diffTimeInNanoSeconds(from: start, to: end)
|
||||
}
|
||||
|
||||
/// Measure the `fn` and return the average sample time per iteration (in c.timeUnit).
|
||||
func measure(_ name: String, fn: (Int) -> Void, numIters: Int) -> Int {
|
||||
#if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER
|
||||
name.withCString { p in startTrackingObjects(p) }
|
||||
#endif
|
||||
|
||||
startMeasurement()
|
||||
fn(numIters)
|
||||
stopMeasurement()
|
||||
|
||||
#if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER
|
||||
name.withCString { p in stopTrackingObjects(p) }
|
||||
#endif
|
||||
|
||||
switch c.timeUnit {
|
||||
case .nanoseconds: return lastSampleTime.nanoseconds / numIters
|
||||
case .microseconds: return lastSampleTime.microseconds / numIters
|
||||
case .milliseconds: return lastSampleTime.milliseconds / numIters
|
||||
case .seconds: return lastSampleTime.seconds / numIters
|
||||
}
|
||||
}
|
||||
|
||||
func logVerbose(_ msg: @autoclosure () -> String) {
|
||||
if c.verbose { print(msg()) }
|
||||
}
|
||||
|
||||
/// Run the benchmark and return the measured results.
|
||||
func run(_ test: BenchmarkInfo) -> BenchResults? {
|
||||
// Before we do anything, check that we actually have a function to
|
||||
// run. If we don't it is because the benchmark is not supported on
|
||||
// the platform and we should skip it.
|
||||
guard let testFn = test.runFunction else {
|
||||
logVerbose("Skipping unsupported benchmark \(test.name)!")
|
||||
return nil
|
||||
}
|
||||
logVerbose("Running \(test.name)")
|
||||
|
||||
var samples: [Int] = []
|
||||
|
||||
func addSample(_ time: Int) {
|
||||
logVerbose(" Sample \(samples.count),\(time)")
|
||||
samples.append(time)
|
||||
}
|
||||
|
||||
resetMeasurements()
|
||||
if let setUp = test.setUpFunction {
|
||||
setUp()
|
||||
stopMeasurement()
|
||||
logVerbose(" SetUp \(lastSampleTime.microseconds)")
|
||||
resetMeasurements()
|
||||
}
|
||||
|
||||
// Determine number of iterations for testFn to run for desired time.
|
||||
func iterationsPerSampleTime() -> (numIters: Int, oneIter: Int) {
|
||||
let oneIter = measure(test.name, fn: testFn, numIters: 1)
|
||||
if oneIter > 0 {
|
||||
let timePerSample = Int(c.sampleTime * 1_000_000.0) // microseconds (μs)
|
||||
return (max(timePerSample / oneIter, 1), oneIter)
|
||||
} else {
|
||||
return (1, oneIter)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine the scale of measurements. Re-use the calibration result if
|
||||
// it is just one measurement.
|
||||
func calibrateMeasurements() -> Int {
|
||||
let (numIters, oneIter) = iterationsPerSampleTime()
|
||||
if numIters == 1 { addSample(oneIter) }
|
||||
else { resetMeasurements() } // for accurate yielding reports
|
||||
return numIters
|
||||
}
|
||||
|
||||
let numIters = min( // Cap to prevent overflow on 32-bit systems when scaled
|
||||
Int.max / 10000, // by the inner loop multiplier inside the `testFn`.
|
||||
c.numIters ?? calibrateMeasurements()
|
||||
)
|
||||
|
||||
let numSamples = c.numSamples ?? min(
|
||||
200, // Cap the number of samples
|
||||
c.numIters == nil ? 1 : calibrateMeasurements()
|
||||
)
|
||||
|
||||
samples.reserveCapacity(numSamples)
|
||||
logVerbose(" Collecting \(numSamples) samples.")
|
||||
logVerbose(" Measuring with scale \(numIters).")
|
||||
for _ in samples.count ..< numSamples {
|
||||
addSample(measure(test.name, fn: testFn, numIters: numIters))
|
||||
}
|
||||
|
||||
test.tearDownFunction?()
|
||||
if let lf = test.legacyFactor {
|
||||
logVerbose(" Applying legacy factor: \(lf)")
|
||||
samples = samples.map { $0 * lf }
|
||||
}
|
||||
|
||||
return BenchResults(samples, maxRSS: measureMemoryUsage())
|
||||
}
|
||||
|
||||
var header: String {
|
||||
let withUnit = { $0 + "(\(self.c.timeUnit))" }
|
||||
let withDelta = { "𝚫" + $0 }
|
||||
func quantiles(q: Int) -> [String] {
|
||||
// See https://en.wikipedia.org/wiki/Quantile#Specialized_quantiles
|
||||
let prefix = [
|
||||
2: "MEDIAN", 3: "T", 4: "Q", 5: "QU", 6: "S", 7: "O", 10: "D",
|
||||
12: "Dd", 16: "H", 20: "V", 33: "TT", 100: "P", 1000: "Pr",
|
||||
][q, default: "\(q)-q"]
|
||||
let base20 = "0123456789ABCDEFGHIJ".map { String($0) }
|
||||
let index: (Int) -> String =
|
||||
{ q == 2 ? "" : q <= 20 ? base20[$0] : String($0) }
|
||||
let tail = (1 ..< q).map { prefix + index($0) } + ["MAX"]
|
||||
return [withUnit("MIN")] + tail.map(c.delta ? withDelta : withUnit)
|
||||
}
|
||||
return (
|
||||
["#", "TEST", "SAMPLES"] +
|
||||
(
|
||||
c.quantile.map(quantiles)
|
||||
?? ["MIN", "MAX", "MEAN", "SD", "MEDIAN"].map(withUnit)
|
||||
) +
|
||||
(c.logMemory ? ["MAX_RSS(B)"] : [])
|
||||
).joined(separator: c.delim)
|
||||
}
|
||||
|
||||
/// Execute benchmarks and continuously report the measurement results.
|
||||
func runBenchmarks() {
|
||||
var testCount = 0
|
||||
|
||||
func report(_ index: String, _ t: BenchmarkInfo, results: BenchResults?) {
|
||||
func values(r: BenchResults) -> [String] {
|
||||
func quantiles(q: Int) -> [Int] {
|
||||
let qs = (0 ... q).map { i in r[Double(i) / Double(q)] }
|
||||
return c.delta ?
|
||||
qs.reduce(into: (encoded: [], last: 0)) {
|
||||
$0.encoded.append($1 - $0.last); $0.last = $1
|
||||
}.encoded : qs
|
||||
}
|
||||
return (
|
||||
[r.sampleCount] +
|
||||
(
|
||||
c.quantile.map(quantiles)
|
||||
?? [r.min, r.max, r.mean, r.sd, r.median]
|
||||
) +
|
||||
[r.maxRSS].compactMap { $0 }
|
||||
).map { (c.delta && $0 == 0) ? "" : String($0) } // drop 0s in deltas
|
||||
}
|
||||
let benchmarkStats = (
|
||||
[index, t.name] + (results.map(values) ?? ["Unsupported"])
|
||||
).joined(separator: c.delim)
|
||||
|
||||
print(benchmarkStats)
|
||||
fflush(stdout)
|
||||
|
||||
if results != nil {
|
||||
testCount += 1
|
||||
}
|
||||
}
|
||||
|
||||
print(header)
|
||||
|
||||
for (index, test) in c.tests {
|
||||
report(index, test, results: run(test))
|
||||
}
|
||||
|
||||
print("\nTotal performance tests executed: \(testCount)")
|
||||
}
|
||||
}
|
||||
|
||||
public func main() {
|
||||
let config = TestConfig(registeredBenchmarks)
|
||||
switch config.action {
|
||||
case .listTests:
|
||||
print("#\(config.delim)Test\(config.delim)[Tags]")
|
||||
for (index, t) in config.tests {
|
||||
let testDescription = [index, t.name, t.tags.sorted().description]
|
||||
.joined(separator: config.delim)
|
||||
print(testDescription)
|
||||
}
|
||||
case .run:
|
||||
TestRunner(config).runBenchmarks()
|
||||
if let x = config.afterRunSleep {
|
||||
sleep(x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
## Swift Benchmark Utils
|
||||
|
||||
> This benchmarking infrastructure is copied from
|
||||
https://github.com/apple/swift/tree/cf53143a47278c2a465409a67376642515956777/benchmark/utils
|
||||
with the intent of producing similar look and feel, as well as because we need some benchmark infra.
|
||||
> When feasible we will aim to collaborate and contribute improvements back to the mainline Swift project.
|
||||
Loading…
Reference in New Issue