put the compare perf script into the repo (#564)
Motivation: For a long while I had a script which runs NIO's performance tests against different Swift toolchains. It should be in the repository. Modifications: add scripts/compare_perf_of_swift_versions.sh Result: everybody can run the performance tests against different Swift versions
This commit is contained in:
parent
a7fee8d98f
commit
8078cc8e04
|
@ -0,0 +1,99 @@
|
||||||
|
#!/bin/bash
|
||||||
|
##===----------------------------------------------------------------------===##
|
||||||
|
##
|
||||||
|
## This source file is part of the SwiftNIO open source project
|
||||||
|
##
|
||||||
|
## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
|
||||||
|
## Licensed under Apache License v2.0
|
||||||
|
##
|
||||||
|
## See LICENSE.txt for license information
|
||||||
|
## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
|
||||||
|
##
|
||||||
|
## SPDX-License-Identifier: Apache-2.0
|
||||||
|
##
|
||||||
|
##===----------------------------------------------------------------------===##
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||||
|
|
||||||
|
function usage() {
|
||||||
|
echo >&2 "$0 SWIFT_BINARY_BASE SWIFT_BINARY_COMPETITOR"
|
||||||
|
echo >&2
|
||||||
|
echo >&2 "Runs the NIOPerformanceTester tool with 'swift' binary "
|
||||||
|
echo >&2 "SWIFT_BINARY_BASE against SWIFT_BINARY_COMPETITOR."
|
||||||
|
echo >&2
|
||||||
|
echo >&2 "Example:"
|
||||||
|
echo >&2 "$0 /Library/Developer/Toolchains/swift-4.1-RELEASE.xctoolchain/usr/bin/swift /Library/Developer/Toolchains/swift-4.2-DEVELOPMENT-SNAPSHOT-2018-08-07-a.xctoolchain/usr/bin/swift"
|
||||||
|
}
|
||||||
|
|
||||||
|
function write_compare_r_script() {
|
||||||
|
cat >> "$1" <<"EOF"
|
||||||
|
#!/usr/bin/env Rscript
|
||||||
|
|
||||||
|
args <- commandArgs(trailingOnly=TRUE)
|
||||||
|
perfBase <- read.csv(args[2], header=F)
|
||||||
|
perfOther <- read.csv(args[4], header=F)
|
||||||
|
nameBase <- args[1]
|
||||||
|
nameOther <- args[3]
|
||||||
|
benchmarks <- perfBase$V2
|
||||||
|
stopifnot(benchmarks == perfOther$V2)
|
||||||
|
|
||||||
|
swifts <- list(nameBase, nameOther)
|
||||||
|
result_cols = 3:12
|
||||||
|
|
||||||
|
perfBase$swift <- rep(swifts[[1]], nrow(perfBase))
|
||||||
|
perfOther$swift <- rep(swifts[[2]], nrow(perfOther))
|
||||||
|
|
||||||
|
get_perf_matrix <- function(.df) {
|
||||||
|
matrix(as.vector(do.call(c, .df[,result_cols])), nrow=nrow(.df))
|
||||||
|
}
|
||||||
|
|
||||||
|
all_perfs <- rbind(perfBase, perfOther)
|
||||||
|
all_perfs$main_indicator <- apply(get_perf_matrix(all_perfs), 1, min)
|
||||||
|
|
||||||
|
for (benchmark in benchmarks) {
|
||||||
|
cat("benchmark:", benchmark, "\n===========\n")
|
||||||
|
p <- all_perfs[all_perfs$V2 == benchmark,]
|
||||||
|
|
||||||
|
for (swift in swifts) {
|
||||||
|
results <- unlist(p[p$swift == swift,][result_cols], use.names=F)
|
||||||
|
|
||||||
|
cat(paste('- with', swift), "\n")
|
||||||
|
cat(results, "\n")
|
||||||
|
print(summary(results))
|
||||||
|
}
|
||||||
|
|
||||||
|
cat(paste("winner: ", p[which.min(p$main_indicator),]$swift), "\n")
|
||||||
|
|
||||||
|
pBase <- p[p$swift == nameBase,]$main_indicator
|
||||||
|
pOther <- p[p$swift == nameOther,]$main_indicator
|
||||||
|
|
||||||
|
cat("comparison:", nameBase, "to", nameOther, ":", round(100 * (pOther - pBase)/pBase, 2), "%\n")
|
||||||
|
cat("\n")
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! test $# -eq 2; then
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
compare_r_script=$(mktemp /tmp/.compare_perf_XXXXXX)
|
||||||
|
write_compare_r_script "$compare_r_script"
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$here/.."
|
||||||
|
compare_args=()
|
||||||
|
for swift_binary in "$@"; do
|
||||||
|
echo "running with $swift_binary"
|
||||||
|
perf_file=$(mktemp /tmp/.nio-perf_XXXXXX)
|
||||||
|
"$swift_binary" run -c release NIOPerformanceTester | grep ^measuring: | tr : , > "$perf_file" 2>&1
|
||||||
|
compare_args+=("$swift_binary")
|
||||||
|
compare_args+=("$perf_file")
|
||||||
|
done
|
||||||
|
|
||||||
|
Rscript "$compare_r_script" "${compare_args[@]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
rm "$compare_r_script"
|
Loading…
Reference in New Issue