Jenkins integration example¶
Runperf allows to integrate with Jenkins via xunit results and also can deliver html results with great level of details especially for regression testing. Let me present one example which is not meant to be copy&pasted to your environment, but can serve as an inspiration for integration. An overview might look like this:
Which might generate following html results (slightly outdated version that compares a different setting on host and guest): here
Let’s imagine we have example.org machine, we can create
rp-example job to run regression testing, then rp-example-manual
job to allow testing of changes or custom params. For each of these
we might want to create $name-ident jobs to allow cherry-picking
and analyzing of results in order to create models to easier evaluate
the expected results.
A useful addition is the rp-analysis job to allow running custom
compare queries without the need to download and run them from your
machines and the rp-prune-artifacts to automatically remove big
tarballs with full results and only keep the json results that are
small and suffice for compare-perf usecases.
The latest addition is an upstream qemu bisect pipeline called
rp-example-upstream-bisect.
The above pipelines are using Jenkins shared library runperf,
which is attached below.
All of these can be easily defined via Jenkins Job Builder:
##############################################################################
# Default configuration
##############################################################################
- defaults:
name: "global"
mailto: ""
wrappers:
- ansicolor
- timestamps
- workspace-cleanup
build-discarder:
days-to-keep: 365
artifact-num-to-keep: 60
# Default runperf params
param-distro: ''
param-guest-distro: ''
param-tests: "'fio:{{\"targets\": \"/fio\"}}' 'uperf:{{\"protocols\": \"tcp\"}}' 'uperf:{{\"protocols\": \"udp\", \"test-types\": \"rr\"}}'"
param-profiles: "Localhost DefaultLibvirt TunedLibvirt"
param-src-build: '1'
param-cmp-tolerance: 5
param-cmp-stddev-tolerance: 10
param-cmp-model-job: ''
param-cmp-model-build: ''
param-host-bkr-links: ''
param-no-reference-builds: 14
param-fio-nbd-setup: false
param-upstream-qemu-commit: ''
param-github-publisher: ''
param-metadata: ''
param-host-script: ''
param-worker-script: ''
trigger-on: "H 17 * * *"
disabled: false
##############################################################################
# Definition for the run-perf execution job
##############################################################################
- job-template:
name: "{name}-run"
triggers:
- timed: "{trigger-on}"
project-type: pipeline
parameters:
- string:
name: DISTRO
description: 'Distribution to be installed/is installed (Fedora-31), when empty latest el8 nightly build is obtained from bkr'
default: "{param-distro}"
- string:
name: GUEST_DISTRO
description: 'Distribution to be installed on guest, when empty "distro" is used'
default: "{param-guest-distro}"
- string:
name: MACHINE
description: 'Machine to be provisioned and tested'
default: "{param-machine}"
- string:
name: ARCH
description: 'Target machine architecture'
default: "{param-arch}"
- string:
name: TESTS
description: 'Space separated list of tests to be executed (WARNING: fio-nbd test requires the FIO_NBD_SETUP checkbox enabled!)'
default: "{param-tests}"
- string:
name: PROFILES
description: 'Space separated list of profiles to be applied'
default: "{param-profiles}"
- string:
name: SRC_BUILD
description: 'Base build to compare with'
default: "{param-src-build}"
- string:
name: CMP_MODEL_JOB
description: 'Job to copy linear "model.json" from'
default: "{param-cmp-model-job}"
- string:
name: CMP_MODEL_BUILD
description: 'Build to copy linear "model.json" from (-1 means lastSuccessful)'
default: "{param-cmp-model-build}"
- string:
name: CMP_TOLERANCE
description: Tolerance for mean values
default: "{param-cmp-tolerance}"
- string:
name: CMP_STDDEV_TOLERANCE
description: Tolerance for standard deviation values
default: "{param-cmp-stddev-tolerance}"
- string:
name: HOST_KERNEL_ARGS
description: Add custom kernel arguments on host
default: ""
- string:
name: HOST_BKR_LINKS
description: 'Space separated list of urls to be grepped for "http.*$arch\\.rpm" and "http.*noarch\\.rpm", filtered by HOST_BKR_LINKS_FILTER and then set to be installed on host/workers with "--allowerase". Intended usage is to use koji/beaker link like: https://koji.fedoraproject.org/koji/buildinfo?buildID=1534744'
default: "{param-host-bkr-links}"
- string:
name: HOST_BKR_LINKS_FILTER
description: 'Filter to be split and applied via "grep -v -e EXPR1 -e EXPR2 ..."'
default: "debug"
- string:
name: GUEST_KERNEL_ARGS
description: Add custom kernel argsuments on workers/guests
default: ""
- string:
name: GUEST_BKR_LINKS
description: 'Space separated list of urls to be grepped for "http.*$arch\\.rpm" and "http.*noarch\\.rpm", filtered by GUEST_BKR_LINKS_FILTER and then set to be installed on host/workers with "--allowerase". Intended usage is to use koji/beaker link like: https://koji.fedoraproject.org/koji/buildinfo?buildID=1534744'
default: ""
- string:
name: GUEST_BKR_LINKS_FILTER
description: 'Filter to be split and applied via "grep -v -e EXPR1 -e EXPR2 ..."'
default: "debug"
- bool:
name: PBENCH_PUBLISH
description: 'Push the pbench results to company pbench server'
default: "{param-pbench-publish}"
- string:
name: GITHUB_PUBLISHER_PROJECT
description: 'Github publisher project ID (when you want to publish your results)'
default: "{param-github-publisher}"
- bool:
name: FIO_NBD_SETUP
description: 'Compile and install fio with nbd ioengine enabled before test execution'
default: "{param-fio-nbd-setup}"
- string:
name: UPSTREAM_QEMU_COMMIT
description: 'Compile and install qemu using provided commit/tag from the upstream git. Use it by using $PROFILE:{{"qemu_bin": "/usr/local/bin/qemu-system-$ARCH"}} when specifying profiles.'
default: "{param-upstream-qemu-commit}"
- bool:
name: FEDORA_LATEST_KERNEL
description: 'Install the latest kernel from koji (Fedora rpm)'
default: false
- string:
name: METADATA
description: 'Additional run-perf --metadata arguments'
default: "{param-metadata}"
- text:
name: HOST_SCRIPT
description: 'Host script to be executed on all --servers'
default: "{param-host-script}"
- text:
name: WORKER_SCRIPT
description: 'Worker script to be executed on all runperf workers'
default: "{param-worker-script}"
- string:
name: DESCRIPTION_PREFIX
description: Description prefix (describe the difference from default)
default: ""
- string:
name: NO_REFERENCE_BUILDS
description: "Number of reference builds for comparison"
default: "{param-no-reference-builds}"
sandbox: true
pipeline-scm:
scm:
- git:
url: git://PATH_TO_YOUR_REPO_WITH_PIPELINES.git
branches:
- main
script-path: "runperf.groovy"
lightweight-checkout: true
##############################################################################
# Definition for the upstream qemu bisect job
##############################################################################
- job-template:
name: "{name}-bisect-qemu"
project-type: pipeline
parameters:
- string:
name: DISTRO
description: 'Distribution to be installed/is installed (Fedora-31), when empty latest el8 nightly build is obtained from bkr'
default: "{param-distro}"
- string:
name: GUEST_DISTRO
description: 'Distribution to be installed on guest, when empty "distro" is used'
default: "{param-guest-distro}"
- string:
name: MACHINE
description: 'Machine to be provisioned and tested'
default: "{param-machine}"
- string:
name: ARCH
description: 'Target machine architecture'
default: "{param-arch}"
- string:
name: TESTS
description: 'Space separated list of tests to be executed (WARNING: fio-nbd test requires the FIO_NBD_SETUP checkbox enabled!)'
default: "{param-tests}"
- string:
name: PROFILES
description: 'Space separated list of profiles to be applied'
default: "{param-profiles}"
- text:
name: HOST_SCRIPT
description: 'Host script to be executed on all --servers'
default: "{param-host-script}"
- text:
name: WORKER_SCRIPT
description: 'Worker script to be executed on all runperf workers'
default: "{param-worker-script}"
- string:
name: HOST_KERNEL_ARGS
description: Add custom kernel arguments on host
default: ""
- string:
name: HOST_BKR_LINKS
description: 'Space separated list of urls to be grepped for "http.*$arch\\.rpm" and "http.*noarch\\.rpm", filtered by HOST_BKR_LINKS_FILTER and then set to be installed on host/workers with "--allowerase". Intended usage is to use koji/beaker link like: https://koji.fedoraproject.org/koji/buildinfo?buildID=1534744'
default: ""
- string:
name: HOST_BKR_LINKS_FILTER
description: 'Filter to be split and applied via "grep -v -e EXPR1 -e EXPR2 ..."'
default: "debug"
- string:
name: GUEST_KERNEL_ARGS
description: Add custom kernel argsuments on workers/guests
default: ""
- string:
name: GUEST_BKR_LINKS
description: 'Space separated list of urls to be grepped for "http.*$arch\\.rpm" and "http.*noarch\\.rpm", filtered by GUEST_BKR_LINKS_FILTER and then set to be installed on host/workers with "--allowerase". Intended usage is to use koji/beaker link like: https://koji.fedoraproject.org/koji/buildinfo?buildID=1534744'
default: ""
- string:
name: GUEST_BKR_LINKS_FILTER
description: 'Filter to be split and applied via "grep -v -e EXPR1 -e EXPR2 ..."'
default: "debug"
- bool:
name: PBENCH_PUBLISH
description: 'Push the pbench results to company pbench server'
default: "{param-pbench-publish}"
- bool:
name: FIO_NBD_SETUP
description: 'Compile and install fio with nbd ioengine enabled before test execution'
default: "{param-fio-nbd-setup}"
- bool:
name: TWO_OUT_OF_THREE
description: 'Use 2 out of 3 result evaluation (longer duration, better stability for jittery results)'
default: true
- string:
name: UPSTREAM_QEMU_GOOD
description: 'SHA of the last good (older) upstream qemu.'
default: "{param-upstream-qemu-commit}"
- string:
name: UPSTREAM_QEMU_BAD
description: 'SHA of the last bad (newer) upstream qemu.'
default: "{param-upstream-qemu-commit}"
- string:
name: DESCRIPTION_PREFIX
description: Description prefix (describe the difference from default)
default: ""
sandbox: true
pipeline-scm:
scm:
- git:
url: git://PATH_TO_YOUR_REPO_WITH_PIPELINES.git
branches:
- main
script-path: "upstream_bisect.groovy"
lightweight-checkout: true
##############################################################################
# Definition for a multi-run-perf execution job
##############################################################################
- job-template:
name: "{name}-multi"
project-type: pipeline
parameters:
- string:
name: JOB_NAME
description: 'Name of the run-perf job to be used for triggering the tests.\nWarning: there might be concurrency issues in case one attempts to concurrently schedule jobs!'
default: "{param-job}"
- string:
name: MACHINE
description: 'Machine to be provisioned and tested'
default: "{param-machine}"
- string:
name: ARCH
description: 'Target machine architecture'
default: "{param-arch}"
- string:
name: TESTS
description: 'Space separated list of tests to be executed (WARNING: fio-nbd test requires the FIO_NBD_SETUP checkbox enabled!)'
default: "{param-tests}"
- string:
name: PROFILES
description: 'Space separated list of profiles to be applied'
default: "{param-profiles}"
- string:
name: CMP_MODEL_JOB
description: 'Job to copy linear "model.json" from'
default: "{param-cmp-model-job}"
- string:
name: CMP_MODEL_BUILD
description: 'Build to copy linear "model.json" from (-1 means lastSuccessful)'
default: "{param-cmp-model-build}"
- string:
name: CMP_TOLERANCE
description: Tolerance for mean values
default: "{param-cmp-tolerance}"
- string:
name: CMP_STDDEV_TOLERANCE
description: Tolerance for standard deviation values
default: "{param-cmp-stddev-tolerance}"
- bool:
name: FIO_NBD_SETUP
description: 'Compile and install fio with nbd ioengine enabled before test execution'
default: "{param-fio-nbd-setup}"
- string:
name: DESCRIPTION_PREFIX
description: Description prefix (describe the difference from default)
default: ""
- bool:
name: PBENCH_PUBLISH
description: 'Push the pbench results to company pbench server'
default: "{param-pbench-publish}"
- string:
name: GITHUB_PUBLISHER_PROJECT
description: 'Github publisher project ID (when you want to publish your results)'
default: ""
- text:
name: HOST_SCRIPT
description: 'Host script to be executed on all --servers'
default: "{param-host-script}"
- text:
name: WORKER_SCRIPT
description: 'Worker script to be executed on all runperf workers'
default: "{param-worker-script}"
- string:
name: NO_ITERATIONS
description: 'How many times to run each iteration'
default: ''
- string:
name: DISTROS
description: '`;` separated list of distributions to be installed/is installed (Fedora-31), when empty latest el8 nightly build is obtained from bkr, when `..` is used it uses bkr to fill all available versions in between the specified versions'
default: "{param-distro}"
- string:
name: GUEST_DISTROS
description: '`;` separated list of distribution to be installed on guest, when empty "distro" is used, when `..` is used it uses bkr to fill all available versions in between the specified versions'
default: "{param-guest-distro}"
- string:
name: HOST_KERNEL_ARGSS
description: '`;` separated list of Add custom kernel arguments on host'
default: ""
- string:
name: HOST_BKR_LINKSS
description: '`;` separated list of space separated lists of urls to be grepped for "http.*$arch\\.rpm" and "http.*noarch\\.rpm", filtered by HOST_BKR_LINKS_FILTER and then set to be installed on host/workers with "--allowerase". Intended usage is to use koji/beaker link like: https://koji.fedoraproject.org/koji/buildinfo?buildID=1534744'
default: "{param-host-bkr-links}"
- string:
name: HOST_BKR_LINKS_FILTER
description: 'Filter to be split and applied via "grep -v -e EXPR1 -e EXPR2 ..."'
default: "debug"
- string:
name: GUEST_KERNEL_ARGSS
description: '`;` separated list of custom kernel argsuments on workers/guests'
default: ""
- string:
name: GUEST_BKR_LINKSS
description: '`;` separated list of space separated lists of urls to be grepped for "http.*$arch\\.rpm" and "http.*noarch\\.rpm", filtered by GUEST_BKR_LINKS_FILTER and then set to be installed on host/workers with "--allowerase". Intended usage is to use koji/beaker link like: https://koji.fedoraproject.org/koji/buildinfo?buildID=1534744'
default: ""
- string:
name: GUEST_BKR_LINKS_FILTER
description: 'Filter to be split and applied via "grep -v -e EXPR1 -e EXPR2 ..."'
default: "debug"
- string:
name: UPSTREAM_QEMU_COMMITS
description: '`;` separated list of qemu commit/tags to be deployed from the upstream git. Use it by using $PROFILE:{{"qemu_bin": "/usr/local/bin/qemu-system-$ARCH"}} when specifying profiles.'
default: "{param-upstream-qemu-commit}"
sandbox: true
pipeline-scm:
scm:
- git:
url: git://PATH_TO_YOUR_REPO_WITH_PIPELINES.git
branches:
- main
script-path: "multi_runperf.groovy"
lightweight-checkout: true
##############################################################################
# Definition of the analyze-perf job
##############################################################################
- job-template:
name: "rp-analysis-{user}"
project-type: pipeline
concurrent: false
description: |
This job allows to cherry-pick results from runperf job and redo the analysis. It is
not thread-safe, therefor it is advised to copy this job with user-suffix and run
the analysis in series storing the graphs manually before submitting next comparison.
parameters:
- string:
name: SRC_JOB
default: "{param-src-job}"
desciption: Source jenkins job
- string:
name: BUILDS
default: ""
description: "List of space separated build numbers to be analyzed, first build is used as source build (not included in graphs)"
- string:
name: DESCRIPTION
default: ""
description: Description of this analysis
- string:
name: CMP_MODEL_JOB
description: 'Job to copy linear "model.json" from'
default: "{param-cmp-model-job}"
- string:
name: CMP_MODEL_BUILD
description: 'Build to copy linear "model.json" from (-1 means lastSuccessful)'
default: "{param-cmp-model-build}"
- string:
name: CMP_TOLERANCE
description: Tolerance for mean values
default: "{param-cmp-tolerance}"
- string:
name: CMP_STDDEV_TOLERANCE
description: Tolerance for standard deviation values
default: "{param-cmp-stddev-tolerance}"
sandbox: true
pipeline-scm:
scm:
- git:
url: git://PATH_TO_YOUR_REPO_WITH_PIPELINES.git
branches:
- main
script-path: "compareperf.groovy"
lightweight-checkout: true
##############################################################################
# Definition of the analyze-perf job
##############################################################################
- job-template:
name: "{name}-identify"
project-type: pipeline
description: |
This job uses analyze-perf script to create model that can be used to better
evaluate run-perf results.
parameters:
- string:
name: SRC_JOB
default: "{name}-run"
desciption: Source jenkins job
- string:
name: BUILDS
default: ""
description: "List of space separated build numbers to be used"
- string:
name: DESCRIPTION
default: ""
description: Free-form description
- string:
name: EXTRA_ARGS
default: ""
description: Additional analyze-perf arguments, for example -t to override default tolerance
- string:
name: REBASE_MODEL_BUILD
description: 'Build number of this job to be used as "--rebase-model" argument'
default: ""
sandbox: true
pipeline-scm:
scm:
- git:
url: git://PATH_TO_YOUR_REPO_WITH_PIPELINES.git
branches:
- main
script-path: "identify.groovy"
lightweight-checkout: true
##############################################################################
# Definition of the prune artifacts job
##############################################################################
- job-template:
name: "rp-prune-artifacts"
node: master
description: |
Remove the big tar.xz files from oldish results not tagged
as keep-forewer.
triggers:
- timed: "H 06 * * *"
parameters:
- string:
name: JOB
default: "{list,your,runperf,jobs,here,to,clean,them,daily}"
description: Name of the job to be pruned
- string:
name: AGE
default: "{param-age}"
description: How old results should be pruned
builders:
- python:
!include-raw-escape: ../scripts/prune_artifacts.py
##############################################################################
# Definition of the git-publisher job
##############################################################################
- job-template:
name: "rp-publish-results-git"
description: |
Publish the build result in git so it can be viewed eg. in github
pages
project-type: pipeline
parameters:
- string:
name: JOB
default: ""
description: "Job containing the result"
- string:
name: BUILD
default: ""
description: "Build of the job with the result"
- bool:
name: STATUS
default: false
description: "Status of the comparison (GOOD=true/BAD=false)"
- string:
name: NOTES
default: ""
description: "Notes to be added as description to the result entry"
- string:
name: PROJECT
default: ""
description: "Owner of the results (usually a group/company name + project/machine)"
- string:
name: TAG
default: "all"
description: "Version tag used to split results of different versions/tags"
- bool:
name: STRIP_RESULTS
default: true
description: "Publish stripped results (MB->KB)"
- string:
name: OS_VERSION
default: ""
description: "Override the os version"
- string:
name: QEMU_SHA
defalut: ""
description: "Override the qemu SHA"
pipeline-scm:
scm:
- git:
url: git://PATH_TO_YOUR_REPO_WITH_PIPELINES.git
branches:
- main
script-path: "publish-results-git.groovy"
lightweight-checkout: true
###############################################################################
## Project to define jobs for automated regression jobs on example.org machine
###############################################################################
#- project:
# name: rp-example
# param-machine: "example.org"
# param-arch: "x86_64"
# param-src-build: 1
# param-cmp-model-job: "{name}-identify"
# param-cmp-model-build: -1
# param-pbench-publish: true
# jobs:
# - "{name}-run"
# - "{name}-identify"
#
#
###############################################################################
## Project to define manual jobs for example.org machine
###############################################################################
#- project:
# name: rp-example-manual
# param-machine: "example.org"
# param-arch: "x86_64"
# param-distro: "YOUR STABLE RELEASE"
# param-src-build: 1
# param-cmp-model-job: "rp-example-manual-identify"
# param-cmp-model-build: 1
# param-pbench-publish: false
# param-job: "{name}-run"
# trigger-on: ""
# jobs:
# - "{name}-run"
# - "{name}-identify"
# - "{name}-multi":
# param-cmp-model-job: ''
# param-cmp-model-build: ''
#
#
###############################################################################
## Project to allow users to run custom queries out of existing results
###############################################################################
#- project:
# name: rp-analysis
# user:
# - virt
# param-src-job: "rp-example-manual"
# param-cmp-model-job: "rp-example-manual-identify"
# param-cmp-model-build: 1
# jobs:
# - "rp-analysis-{user}"
#
###############################################################################
## Prune artifacts after 14 days, hopefully we would notice and mark/move
## them when full details are needed.
###############################################################################
#- project:
# name: rp-prune-artifacts
# param-age: 14
# jobs:
# - "rp-prune-artifacts"
Now let’s have a look at the runperf.groovy pipeline:
// Pipeline to run runperf and compare to given results
// groovylint-disable-next-line
@Library('runperf') _
// Following `params` have to be defined in job (eg. via jenkins-job-builder)
// Machine to be provisioned and tested
machine = params.MACHINE.trim()
// target machine's architecture
arch = params.ARCH.trim()
// Distribution to be installed/is installed (Fedora-32)
// when empty it will pick the latest available nightly el8
_distro = params.DISTRO.trim()
_distro = _distro ?: 'latest-RHEL-8.0%.n.%'
// Distribution to be installed on guest, when empty "distro" is used
guestDistro = params.GUEST_DISTRO.trim()
// Space separated list of tests to be executed
tests = params.TESTS.trim()
// Space separated list of profiles to be applied
profiles = params.PROFILES.trim()
// Base build to compare with
srcBuild = params.SRC_BUILD.trim()
// Compareperf tollerances
cmpModelJob = params.CMP_MODEL_JOB.trim()
cmpModelBuild = params.CMP_MODEL_BUILD.trim()
cmpTolerance = params.CMP_TOLERANCE.trim()
cmpStddevTolerance = params.CMP_STDDEV_TOLERANCE.trim()
// Add custom kernel arguments on host
hostKernelArgs = params.HOST_KERNEL_ARGS.trim()
// Install rpms from (beaker) urls
hostBkrLinks = params.HOST_BKR_LINKS.trim()
// filters for hostBkrLinks
hostBkrLinksFilter = params.HOST_BKR_LINKS_FILTER.trim()
// Add custom kernel argsuments on workers/guests
guestKernelArgs = params.GUEST_KERNEL_ARGS.trim()
// Install rpms from (beaker) urls
guestBkrLinks = GUEST_BKR_LINKS.trim()
// filters for guestBkrLinks
guestBkrLinksFilter = params.GUEST_BKR_LINKS_FILTER.trim()
// Add steps to fetch, compile and install the upstream fio with nbd ioengine compiled in
fioNbdSetup = params.FIO_NBD_SETUP
// Add steps to checkout, compile and install the upstream qemu from git
upstreamQemuCommit = params.UPSTREAM_QEMU_COMMIT.trim()
// Add steps to install the latest kernel from koji (Fedora rpm)
fedoraLatestKernel = params.FEDORA_LATEST_KERNEL
// Description prefix (describe the difference from default)
descriptionPrefix = params.DESCRIPTION_PREFIX
// Number of reference builds
noReferenceBuilds = params.NO_REFERENCE_BUILDS.toInteger()
// Pbench-publish related options
pbenchPublish = params.PBENCH_PUBLISH
// Github-publisher project ID
githubPublisherProject = params.GITHUB_PUBLISHER_PROJECT.trim()
githubPublisherTag = ''
// Additional run-perf metadata
metadata = params.METADATA
// Custom host/guest setups cript
hostScript = params.HOST_SCRIPT
workerScript = params.WORKER_SCRIPT
// Extra variables
// Provisioner machine
workerNode = 'runperf-slave'
// runperf git branch
gitBranch = 'main'
// extra runperf arguments
extraArgs = ''
node(workerNode) {
stage('Preprocess') {
(distro, guestDistro, descriptionPrefix) = runperf.preprocessDistros(_distro, guestDistro,
arch, descriptionPrefix)
currentBuild.description = "${distro} - in progress"
}
stage('Measure') {
runperf.deployDownstreamConfig(gitBranch)
runperf.deployRunperf(gitBranch)
// Use grubby to update default args on host
hostScript = runperf.setupScript(hostScript, hostKernelArgs, hostBkrLinks, hostBkrLinksFilter,
arch, fioNbdSetup)
workerScript = runperf.setupScript(workerScript, guestKernelArgs, guestBkrLinks, guestBkrLinksFilter,
arch, fioNbdSetup)
// Build custom qemu
if (upstreamQemuCommit) {
// Always translate the user input into the actual commit and also get the description
sh 'rm -Rf upstream_qemu'
dir('upstream_qemu') {
sh 'git clone --filter=tree:0 https://gitlab.com/qemu-project/qemu.git .'
upstreamQemuVersion = sh(returnStdout: true,
script: "git rev-parse ${upstreamQemuCommit}").trim()
githubPublisherTag = sh(returnStdout: true,
script: "git describe --tags --always ${upstreamQemuCommit}"
).trim().split('-')[0]
println("Using qemu $githubPublisherTag commit $upstreamQemuVersion")
}
sh '\\rm -Rf upstream_qemu'
hostScript += '\n\n# UPSTREAM_QEMU_SETUP'
hostScript += '\nOLD_PWD="$PWD"'
hostScript += '\ndnf install --skip-broken -y python3-devel zlib-devel gtk3-devel glib2-static '
hostScript += 'spice-server-devel usbredir-devel make gcc libseccomp-devel numactl-devel '
hostScript += 'libaio-devel git ninja-build'
hostScript += '\ncd /root'
hostScript += '\n[ -e "qemu" ] || { mkdir qemu; cd qemu; git init; git remote add origin '
hostScript += 'https://gitlab.com/qemu-project/qemu.git; cd ..; }'
hostScript += '\ncd qemu'
hostScript += "\ngit fetch --depth=1 origin ${upstreamQemuVersion}"
hostScript += "\ngit checkout -f ${upstreamQemuVersion}"
hostScript += '\ngit submodule update --init'
hostScript += '\nVERSION=$(git rev-parse HEAD)'
hostScript += '\ngit diff --quiet || VERSION+="-dirty"'
hostScript += '\n./configure --target-list="$(uname -m)"-softmmu --disable-werror --enable-kvm '
hostScript += '--enable-vhost-net --enable-attr --enable-fdt --enable-vnc --enable-seccomp '
hostScript += '--enable-usb-redir --disable-opengl --disable-virglrenderer '
hostScript += '--with-pkgversion="$VERSION"'
hostScript += runperf.makeInstallCmd
hostScript += '\nchcon -Rt qemu_exec_t /usr/local/bin/qemu-system-"$(uname -m)"'
hostScript += '\n\\cp -f build/config.status /usr/local/share/qemu/'
hostScript += '\ncd $OLD_PWD'
}
// Install the latest kernel from koji (Fedora rpm)
if (fedoraLatestKernel) {
kernelBuild = sh(returnStdout: true,
script: ("curl '$runperf.kojiUrl/packageinfo?packageID=8' | " +
'grep -B 4 "complete" | grep "kernel" | ' +
'grep "git" | grep -m 1 -o -e \'href="[^"]*"\'')
).trim()[6..-2]
kernelBuildUrl = runperf.kojiUrl + kernelBuild
kernelBuildFilter = 'debug bpftool kernel-tools perf kernel-selftests kernel-doc'
hostScript += runperf.getBkrInstallCmd(kernelBuildUrl, kernelBuildFilter, arch)
workerScript += runperf.getBkrInstallCmd(kernelBuildUrl, kernelBuildFilter, arch)
}
if (hostScript) {
writeFile file: 'host_script', text: hostScript
extraArgs += ' --host-setup-script host_script --host-setup-script-reboot'
}
if (workerScript) {
writeFile file: 'worker_script', text: workerScript
extraArgs += ' --worker-setup-script worker_script'
}
if (pbenchPublish) {
metadata += ' pbench_server_publish=yes'
}
// Using jenkins locking to prevent multiple access to a single machine
lock(machine) {
sh '$KINIT'
status = sh(returnStatus: true,
script: "python3 scripts/run-perf ${extraArgs} -v --hosts ${machine} --distro ${distro} " +
"--provisioner Beaker --default-password YOUR_DEFAULT_PASSWORD --profiles ${profiles} " +
'--log run.log --paths ./downstream_config --metadata ' +
"'build=${currentBuild.number}${descriptionPrefix}' " +
"'url=${currentBuild.absoluteUrl}' 'project=YOUR_PROJECT_ID ${currentBuild.projectName}' " +
"'pbench_server=YOUR_PBENCH_SERVER_URL' " +
"'machine_url_base=https://YOUR_BEAKER_URL/view/%(machine)s' " +
"${metadata} -- ${tests}")
}
// Add new-line after runperf output (ignore error when does not exists
sh(returnStatus: true, script: "echo >> \$(echo -n result*)/RUNPERF_METADATA")
stage('Archive results') {
// Archive only "result_*" as we don't want to archive "resultsNoArchive"
sh returnStatus: true, script: 'tar cf - result_* | xz -T2 -7e - > "$(echo result_*)".tar.xz'
archiveArtifacts allowEmptyArchive: true, artifacts: runperf.runperfArchiveFilter
}
if (status) {
runperf.tryOtherDistros(_distro, arch)
runperf.failBuild('Run-perf execution failed',
"run-perf returned non-zero status ($status)",
distro)
}
}
stage('Compare') {
// Get up to noReferenceBuilds json results to use as a reference
referenceBuilds = []
for (build in runperf.getGoodBuildNumbers(env.JOB_NAME)) {
copyArtifacts(filter: runperf.runperfResultsFilter, optional: true,
fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific("${build}"),
target: "reference_builds/${build}/")
if (findFiles(glob: "reference_builds/${build}/result*/*/*/*/*.json")) {
referenceBuilds.add("${build}:" + sh(returnStdout: true,
script: "echo reference_builds/${build}/*").trim())
if (referenceBuilds.size() >= noReferenceBuilds) {
break
}
}
}
// Get src build's json results to compare against
copyArtifacts(filter: runperf.runperfResultsFilter, optional: true,
fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(srcBuild),
target: 'src_result/')
// If model build set get the model from it's job
if (cmpModelBuild) {
if (cmpModelBuild == '-1') {
copyArtifacts(filter: runperf.modelJson, optional: false, fingerprintArtifacts: true,
projectName: cmpModelJob, selector: lastSuccessful(), target: runperf.thisPath)
} else {
copyArtifacts(filter: runperf.modelJson, optional: false, fingerprintArtifacts: true,
projectName: cmpModelJob, selector: specific(cmpModelBuild), target: runperf.thisPath)
}
cmpExtra = '--model-linear-regression ' + runperf.modelJson
} else {
cmpExtra = ''
}
// Compare the results and generate html as well as xunit results
status = sh(returnStatus: true,
script: ('python3 scripts/compare-perf --log compare.log ' +
'--tolerance ' + cmpTolerance + ' --stddev-tolerance ' + cmpStddevTolerance +
" --xunit ${runperf.resultXml} --html ${runperf.htmlIndex} --html-small-file " + cmpExtra +
' -- src_result/* ' + referenceBuilds.reverse().join(' ') +
' $(find . -maxdepth 1 -type d ! -name "*.tar.*" -name "result*")'))
if (fileExists(runperf.resultXml)) {
if (status) {
// This could mean there were no tests to compare or other failures, interrupt the build
echo "Non-zero exit status: ${status}"
}
} else {
runperf.failBuild('Compare-perf execution failed',
"Missing ${runperf.resultXml}, exit code: ${status}",
distro)
}
}
stage('Postprocess') {
// Build description
currentBuild.description = "${descriptionPrefix}${srcBuild} ${currentBuild.number} ${distro}"
// Store and publish html results
archiveArtifacts allowEmptyArchive: true, artifacts: runperf.htmlIndex
if (fileExists(runperf.htmlPath)) {
publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: runperf.htmlPath,
reportFiles: runperf.htmlFile, reportName: 'HTML Report', reportTitles: ''])
}
// Junit results
junit allowEmptyResults: true, testResults: runperf.resultXml
// Remove the unnecessary big files
sh runperf.runperfArchFilterRmCmd
// Publish the results
if (githubPublisherProject) {
build(job: 'rp-publish-results-git',
parameters: [string(name: 'JOB', value: env.JOB_NAME),
string(name: 'BUILD', value: env.BUILD_NUMBER),
booleanParam(name: 'STATUS', value: status == 0),
string(name: 'NOTES', value: descriptionPrefix),
string(name: 'PROJECT', value: githubPublisherProject),
string(name: 'TAG', value: githubPublisherTag),
booleanParam(name: 'STRIP_RESULTS', value: true)],
quietPeriod: 0,
wait: false)
}
}
}
Following compareperf.groovy pipeline is extremely useful for later
analysis, or extra comparison of manual pipelines:
// Pipeline to create comparison of previously generated runperf results
// Following `params` have to be defined in job (eg. via jenkins-job-builder)
// groovylint-disable-next-line
@Library('runperf') _
// Source jenkins job
srcJob = params.SRC_JOB.trim()
// List of space separated build numbers to be analyzed, first build is used
// as source build (not included in graphs)
builds = params.BUILDS.split().toList()
// Description of this analysis
description = params.DESCRIPTION
// Compareperf tollerances
cmpModelJob = params.CMP_MODEL_JOB.trim()
cmpModelBuild = params.CMP_MODEL_BUILD.trim()
cmpTolerance = params.CMP_TOLERANCE.trim()
cmpStddevTolerance = params.CMP_STDDEV_TOLERANCE.trim()
// Extra variables
// Provisioner machine
workerNode = 'runperf-slave'
// runperf git branch
gitBranch = 'main'
// misc variables
thisPath = '.'
spaceChr = ' '
lastBuildChr = '-1'
stage('Analyze') {
node(workerNode) {
assert builds.size() >= 2
runperf.deployRunperf(gitBranch)
referenceBuilds = []
// Get all the reference builds (second to second-to-last ones)
if (builds.size() > 2) {
for (build in builds[1..-2]) {
copyArtifacts(filter: runperf.runperfResultsFilter, optional: true,
fingerprintArtifacts: true, projectName: srcJob, selector: specific(build),
target: "reference_builds/${build}/")
if (fileExists("reference_builds/${build}")) {
referenceBuilds.add("${build}:" + sh(returnStdout: true,
script: "echo reference_builds/${build}/*").trim())
} else {
echo "Skipping reference build ${build}, failed to copy artifacts."
}
}
}
// Get the source build
copyArtifacts(filter: runperf.runperfResultsFilter, optional: false,
fingerprintArtifacts: true, projectName: srcJob, selector: specific(builds[0]),
target: 'src_result/')
// Get the destination build
copyArtifacts(filter: runperf.runperfResultsFilter, optional: false,
fingerprintArtifacts: true, projectName: srcJob, selector: specific(builds[-1]),
target: thisPath)
// Get the model
if (cmpModelBuild) {
if (cmpModelBuild == lastBuildChr) {
copyArtifacts(filter: runperf.modelJson, optional: false, fingerprintArtifacts: true,
projectName: cmpModelJob, selector: lastSuccessful(), target: thisPath)
} else {
copyArtifacts(filter: runperf.modelJson, optional: false, fingerprintArtifacts: true,
projectName: cmpModelJob, selector: specific(cmpModelBuild), target: thisPath)
}
cmpExtra = '--model-linear-regression ' + runperf.modelJson
} else {
cmpExtra = ''
}
status = 0
lock(workerNode) {
// Avoid modifying workerNode's environment while executing compareperf
sh runperf.pythonDeployCmd
status = sh(returnStatus: true,
script: ('python3 scripts/compare-perf -vvv --tolerance ' + cmpTolerance +
' --stddev-tolerance ' + cmpStddevTolerance +
' --xunit ' + runperf.resultXml + ' --html ' + runperf.htmlIndex + spaceChr +
cmpExtra + ' -- src_result/* ' + referenceBuilds.join(spaceChr) +
' $(find . -maxdepth 1 -type d ! -name "*.tar.*" -name "result*")'))
}
if (fileExists(runperf.resultXml)) {
if (status) {
// This could mean there were no tests to compare or other failures, interrupt the build
echo "Non-zero exit status: ${status}"
}
} else {
currentBuild.result = 'FAILED'
error "Missing ${runperf.resultXml}, exit code: ${status}"
}
currentBuild.description = "${description}${builds} ${srcJob}"
archiveArtifacts allowEmptyArchive: true, artifacts: runperf.htmlIndex
junit allowEmptyResults: true, testResults: runperf.resultXml
if (fileExists(runperf.htmlPath)) {
publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: runperf.htmlPath,
reportFiles: runperf.htmlFile, reportName: 'HTML Report', reportTitles: ''])
}
// Remove the unnecessary big files
sh '\\rm -Rf result* src_result* reference_builds'
}
}
And the identify.groovy to allow creating linear models:
// Pipeline to create comparison of previously generated runperf results
// groovylint-disable-next-line
@Library('runperf') _
// Following `params` have to be defined in job (eg. via jenkins-job-builder)
// Source jenkins job
srcJob = params.SRC_JOB
// List of space separated build numbers to be analyzed, first build is used
// as source build (not included in graphs)
builds = params.BUILDS.split().toList()
// Description of this analysis
description = params.DESCRIPTION
// Extra AnalyzePerf arguments
extraArgs = params.EXTRA_ARGS
// Build number used for --rebase-model
rebaseModelBuild = params.REBASE_MODEL_BUILD
// Extra variables
// Provisioner machine
workerNode = 'runperf-slave'
// runperf git branch
gitBranch = 'main'
// misc variables
spaceChr = ' '
stage('Analyze') {
node(workerNode) {
runperf.deployRunperf(gitBranch)
// Get all the specified builds
for (build in builds) {
copyArtifacts(filter: 'result*/**/result*.json', optional: false, fingerprintArtifacts: true,
projectName: srcJob, selector: specific(build), target: 'results/')
}
// If rebaseModel set, get the model from that build
if (rebaseModelBuild) {
copyArtifacts(filter: runperf.modelJson, optional: false, fingerprintArtifacts: true,
projectName: env.JOB_NAME, selector: specific(rebaseModelBuild),
target: 'src_model/')
extraArgs += " --rebase-model 'src_model/$runperf.modelJson'"
}
status = 0
lock(workerNode) {
// Avoid modifying workerNode's environment while executing compareperf
sh 'python3 setup.py develop --user'
status = sh(returnStatus: true,
script: ('python3 scripts/analyze-perf -vvv --stddev-linear-regression ' +
runperf.modelJson + spaceChr + extraArgs + ' -- results/*'))
}
if (fileExists(runperf.modelJson)) {
// This could mean there were no tests to compare or other failures, interrupt the build
if (status) {
echo "Non-zero exit status: ${status}"
}
} else {
currentBuild.result = 'FAILED'
error "Missing ${runperf.modelJson}, exit code: ${status}"
}
if (description) {
currentBuild.description = description + spaceChr + builds.join(spaceChr)
} else {
currentBuild.description = builds.join(spaceChr)
}
archiveArtifacts allowEmptyArchive: true, artifacts: runperf.modelJson
sh '\\rm -Rf results*'
}
}
The cleanup job prune_artifacts.py:
#!/bin/env python3
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright: Red Hat Inc. 2020
# Author: Lukas Doktor <ldoktor@redhat.com>
"""
When executed on a jenkins master it allows to walk the results and remove
"*.tar.*" files on older builds that are not manually marked as
keep-for-infinity.
"""
import time
import glob
import os
import re
JENKINS_DIR = "/var/lib/jenkins/jobs/"
def prune_result(path, before):
"""
Prune result if older than age and keep forever not set
"""
build_path = os.path.join(path, "build.xml")
if not os.path.exists(build_path):
print("KEEP %s - no build.xml" % path)
return
treated_path = os.path.join(path, "ld_artifact_pruned")
if os.path.exists(treated_path):
print("SKIP %s - already treated" % path)
return
with open(build_path) as build_fd:
build_xml = build_fd.read()
if "<keepLog>false</keepLog>" not in build_xml:
print("KEEP %s - keep forever set" % path)
return
match = re.findall(r"<startTime>(\d+)</startTime>", build_xml)
if not match:
print("KEEP %s - no startTime\n%s" % (path, build_xml))
return
start_time = int(match[-1])
if start_time > before:
print("KEEP %s - younger than %s (%s)" % (path, before, start_time))
return
print("PRUNE %s (%s)" % (path, start_time))
for pth in glob.glob(os.path.join(path, "archive", "*.tar.*")):
os.unlink(pth)
with open(treated_path, 'wb'):
"""touching the file"""
def prune_results(job, age):
"""
Walk job's builds and prune them
"""
if not job:
print("No job specified, returning")
return
# Jenkins stores startTime * 1000
before = int((time.time() - age) * 1000)
print("Pruning %s builds older than %s" % (job, before))
builds = glob.glob(os.path.join(JENKINS_DIR, job, "builds", "*"))
for build in builds:
prune_result(build, before)
print("Done")
if __name__ == '__main__':
prune_results(os.environ.get('JOB'),
int(os.environ.get('AGE', 14)) * 86400)
And a bisect job upstream_bisect.groovy:
// Pipeline to run runperf and compare to given results
// groovylint-disable-next-line
@Library('runperf') _
// Following `params` have to be defined in job (eg. via jenkins-job-builder)
// Machine to be provisioned and tested
machine = params.MACHINE.trim()
// target machine's architecture
arch = params.ARCH.trim()
// Distribution to be installed/is installed (Fedora-32)
// when empty it will pick the latest available nightly el8
distro = params.DISTRO.trim()
// Distribution to be installed on guest, when empty "distro" is used
guestDistro = params.GUEST_DISTRO.trim()
// Space separated list of tests to be executed
tests = params.TESTS.trim()
// Space separated list of profiles to be applied
profiles = params.PROFILES.trim()
// Add custom kernel arguments on host
hostKernelArgs = params.HOST_KERNEL_ARGS.trim()
// Install rpms from (beaker) urls
hostBkrLinks = params.HOST_BKR_LINKS.trim()
// filters for hostBkrLinks
hostBkrLinksFilter = params.HOST_BKR_LINKS_FILTER.trim()
// Add custom kernel argsuments on workers/guests
guestKernelArgs = params.GUEST_KERNEL_ARGS.trim()
// Install rpms from (beaker) urls
guestBkrLinks = GUEST_BKR_LINKS.trim()
// filters for guestBkrLinks
guestBkrLinksFilter = params.GUEST_BKR_LINKS_FILTER.trim()
// Add steps to fetch, compile and install the upstream fio with nbd ioengine compiled in
fioNbdSetup = params.FIO_NBD_SETUP
// Specify the bisection range
// Older commit
upstreamQemuGood = params.UPSTREAM_QEMU_GOOD.trim()
// Newer commit
upstreamQemuBad = params.UPSTREAM_QEMU_BAD.trim()
// Description prefix (describe the difference from default)
descriptionPrefix = params.DESCRIPTION_PREFIX
// Pbench-publish related options
pbenchPublish = params.PBENCH_PUBLISH
// Custom host/guest setups cript
hostScript = params.HOST_SCRIPT
workerScript = params.WORKER_SCRIPT
// Extra variables
// Provisioner machine
workerNode = 'runperf-slave'
// runperf git branch
gitBranch = 'main'
// extra runperf arguments
extraArgs = ''
String getBkrInstallCmd(String hostBkrLinks, String hostBkrLinksFilter, String arch) {
return ('\nfor url in ' + hostBkrLinks + '; do dnf install -y --allowerasing ' +
'$(curl -k \$url | grep -o -e "http[^\\"]*' + arch + '\\.rpm" -e ' +
'"http[^\\"]*noarch\\.rpm" | grep -v $(for expr in ' + hostBkrLinksFilter + '; do ' +
'echo -n " -e $expr"; done)); done')
}
node(workerNode) {
stage('Preprocess') {
(distro, guestDistro, descriptionPrefix) = runperf.preprocessDistros(distro, guestDistro,
arch, descriptionPrefix)
currentBuild.description = "${distro} - in progress"
}
stage('Measure') {
runperf.deployDownstreamConfig(gitBranch)
runperf.deployRunperf(gitBranch)
metadata = ''
hostScript = runperf.setupScript(hostScript, hostKernelArgs, hostBkrLinks, hostBkrLinksFilter,
arch, fioNbdSetup)
workerScript = runperf.setupScript(workerScript, guestKernelArgs, guestBkrLinks, guestBkrLinksFilter,
arch, fioNbdSetup)
if (hostScript) {
writeFile file: 'host_script', text: hostScript
extraArgs += ' --host-setup-script host_script --host-setup-script-reboot'
}
if (workerScript) {
writeFile file: 'worker_script', text: workerScript
extraArgs += ' --worker-setup-script worker_script'
}
if (pbenchPublish) {
metadata += ' pbench_server_publish=yes'
}
// Using jenkins locking to prevent multiple access to a single machine
lock(machine) {
// Make sure we have the full upstream_qemu cloned (we don't need submodules, thought)
sh 'rm -Rf upstream_qemu/'
sh 'git clone https://gitlab.com/qemu-project/qemu.git upstream_qemu/'
sh '$KINIT'
// First run the provisioning and dummy test to age the machine a bit
sh("python3 scripts/run-perf ${extraArgs} -v --hosts ${machine} --distro ${distro} " +
'--provisioner Beaker --default-password YOUR_DEFAULT_PASSWORD ' +
'--profiles DefaultLibvirt --paths ./downstream_config --log prejob.log -- ' +
'\'fio:{"runtime": "30", "targets": "/fio", "block-sizes": "4", "test-types": "read", ' +
'"samples": "1"}\'')
// And now run the bisection without reprovisioning
sh("DIFFPERF='python3 scripts/diff-perf' contrib/upstream_qemu_bisect.sh upstream_qemu/ " +
"${upstreamQemuGood} ${upstreamQemuBad} python3 scripts/run-perf ${extraArgs} " +
"-v --hosts ${machine} --distro ${distro} --log job.log " +
"--default-password YOUR_DEFAULT_PASSWORD --profiles ${profiles} " +
"--paths ./downstream_config --metadata " +
"'project=virt-perf-ci ${currentBuild.projectName}' " +
"'pbench_server=YOUR_PBENCH_SERVER_URL' " +
"'machine_url_base=https://YOUR_BEAKER_URL/view/%(machine)s' " +
"${metadata} -- ${tests}")
}
}
stage('Postprocess') {
// Build description
currentBuild.description = "${descriptionPrefix} ${currentBuild.number} ${distro}"
// Move results to mimic usual run-perf results path
if (fileExists('.diff-perf/report.html')) {
diffReportPath = 'html/index.html'
sh('mkdir -p html')
sh("mv '.diff-perf/report.html' '$diffReportPath'")
// Store and publish html results
archiveArtifacts allowEmptyArchive: true, artifacts: diffReportPath
publishHTML([allowMissing: true, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'html/',
reportFiles: 'index.html', reportName: 'HTML Report', reportTitles: ''])
}
// Remove the unnecessary big files
sh 'contrib/bisect.sh clean'
}
}
And a range job to trigger multiple runperf jobs:
// Pipeline to trigger a series of run-perf jobs to cover a range of params.
// Following `params` have to be defined in job (eg. via jenkins-job-builder)
// groovylint-disable-next-line
@Library('runperf') _
csvSeparator = ';'
// SHARED VALUES FOR ALL JOBS
// Job name to be triggered
jobName = params.JOB_NAME.trim()
// Machine to be provisioned and tested
machine = params.MACHINE.trim()
// target machine's architecture
arch = params.ARCH.trim()
// Space separated list of tests to be executed
tests = params.TESTS.trim()
// Space separated list of profiles to be applied
profiles = params.PROFILES.trim()
// Compareperf tollerances
cmpModelJob = params.CMP_MODEL_JOB.trim()
cmpModelBuild = params.CMP_MODEL_BUILD.trim()
cmpTolerance = params.CMP_TOLERANCE.trim()
cmpStddevTolerance = params.CMP_STDDEV_TOLERANCE.trim()
// Add steps to fetch, compile and install the upstream fio with nbd ioengine compiled in
fioNbdSetup = params.FIO_NBD_SETUP
// Description prefix (describe the difference from default)
descriptionPrefix = params.DESCRIPTION_PREFIX
// Pbench-publish related options
pbenchPublish = params.PBENCH_PUBLISH
// Github-publisher project ID
githubPublisherProject = params.GITHUB_PUBLISHER_PROJECT.trim()
// LIST OF VALUES
// Iterations of each combination
if (params.NO_ITERATIONS) {
iterations = 1..params.NO_ITERATIONS.toInteger()
} else {
iterations = [1]
}
// Distribution to be installed/is installed (Fedora-32)
// when empty it will pick the latest available nightly el8
distrosRaw = params.DISTROS.split(csvSeparator)
// Distribution to be installed on guest, when empty "distro" is used
guestDistrosRaw = params.GUEST_DISTROS.split(csvSeparator)
// Add custom kernel arguments on host
hostKernelArgss = params.HOST_KERNEL_ARGSS.split(csvSeparator)
// Install rpms from (beaker) urls
hostBkrLinkss = params.HOST_BKR_LINKSS.split(csvSeparator)
// filters for hostBkrLinks
hostBkrLinksFilter = params.HOST_BKR_LINKS_FILTER
// Add custom kernel argsuments on workers/guests
guestKernelArgss = params.GUEST_KERNEL_ARGSS.split(csvSeparator)
// Install rpms from (beaker) urls
guestBkrLinkss = GUEST_BKR_LINKSS.split(csvSeparator)
// filters for guestBkrLinks
guestBkrLinksFilter = params.GUEST_BKR_LINKS_FILTER
// Add steps to checkout, compile and install the upstream qemu from git
upstreamQemuCommits = params.UPSTREAM_QEMU_COMMITS.split(csvSeparator)
// Custom host/guest setups cript
hostScript = params.HOST_SCRIPT
workerScript = params.WORKER_SCRIPT
// Extra variables
// Provisioner machine
workerNode = 'runperf-slave'
// misc variables
srcBuildUnset = '-1'
distros = runperf.getDistrosRange(distrosRaw, workerNode, arch)
guestDistros = runperf.getDistrosRange(guestDistrosRaw, workerNode, arch)
referenceBuilds = 0
srcBuild = srcBuildUnset
paramTypes = [iterations, guestBkrLinkss, guestKernelArgss, hostBkrLinkss, hostKernelArgss,
upstreamQemuCommits, guestDistros, distros]
for (params in paramTypes.combinations()) {
println("Triggering with: $params")
if (params[0] == 1) {
prefix = descriptionPrefix
} else {
prefix = "${descriptionPrefix}${params[0]}"
}
// TODO: Add no-provisioning-version
// Use a cleanup job to remove host-setup-script things
srcBuild = runperf.triggerRunperf(env.JOB_NAME, srcBuild == srcBuildUnset, params[7], params[6],
machine, arch, tests, profiles, srcBuild, params[4],
params[3], hostBkrLinksFilter, params[2], params[1],
guestBkrLinksFilter, params[5], prefix, pbenchPublish,
fioNbdSetup, Math.max(0, referenceBuilds).toString(),
cmpModelJob, cmpModelBuild, cmpTolerance, cmpStddevTolerance,
githubPublisherProject, hostScript, workerScript)
referenceBuilds += 1
}
To enable the Jenkins shared libraries look at Extending with Shared Libraries tutorial. Once you have your repo enabled, place the following file:
import groovy.transform.Field
// Use this by adding: @Library('runperf') _
// misc variables
@Field String resultXml = 'result.xml'
@Field String htmlPath = 'html'
@Field String htmlFile = 'index.html'
@Field String htmlIndex = "${htmlPath}/${htmlFile}"
@Field String modelJson = 'model.json'
@Field String thisPath = '.'
@Field String runperfArchiveFilter = ('result*/*/*/*/*.json,result*/RUNPERF_METADATA,result*/**/__error*__/**,' +
'result*/**/__sysinfo*__/**,result_*.tar.xz,*.log')
@Field String runperfArchFilterRmCmd = "\\rm -Rf result* src_result* reference_builds ${htmlPath} *.log"
@Field String runperfResultsFilter = 'result*/*/*/*/*.json,result*/RUNPERF_METADATA,result*/**/__error*__/**'
@Field String makeInstallCmd = '\nmake -j $(getconf _NPROCESSORS_ONLN)\nmake install'
@Field String pythonDeployCmd = 'python3 setup.py develop --user'
@Field String kojiUrl = 'https://koji.fedoraproject.org/koji/'
@Field fioNbdScript = ('\n\n# FIO_NBD_SETUP' +
'\ndnf install --skip-broken -y fio gcc zlib-devel libnbd-devel make qemu-img libaio-devel' +
'\ncd /tmp' +
'\ncurl -L https://github.com/axboe/fio/archive/fio-3.19.tar.gz | tar xz' +
'\ncd fio-fio-3.19' +
'\n./configure --enable-libnbd\n' +
makeInstallCmd)
@Field String labController = 'ENTER_LAB_CONTROLLER_URL'
@Field String ownerEmail = 'ENTER_OPERATOR_EMAIL_ADDR
void failBuild(String subject, String details, String distro=distro) {
// Set description, send email and raise exception
currentBuild.description = "BAD ${distro} - $details"
mail(to: ownerEmail,
subject: "${env.JOB_NAME}: $subject",
body: "Job: ${env.BUILD_URL}\n\n$details")
error details
}
List preprocessDistros(String distro, String guestDistro, String arch, descriptionPrefix) {
// Parses the distro and guestDistro params into actual [distro, guestDistro]
if (distro.startsWith('latest-')) {
if (distro.startsWith('latest-untested-')) {
distro = getLatestUntestedDistro(distro[16..-1], arch)
descriptionPrefix += 'U' // To emphasize we use "untested" distros
echo "Using latest-untested distro ${distro} from bkr"
} else {
distro = getLatestDistros(distro[7..-1], 1, arch)[0]
echo "Using latest distro ${distro} from bkr"
}
} else {
echo "Using distro ${distro} from params"
}
if (!guestDistro) {
guestDistro == distro
}
if (guestDistro == distro) {
echo "Using the same guest distro ${distro}"
} else {
echo "Using different guest distro: ${guestDistro} from host: ${distro}"
}
return [distro, guestDistro, descriptionPrefix]
}
void deployRunperf(gitBranch) {
git branch: gitBranch, url: 'https://github.com/distributed-system-analysis/run-perf.git'
// Remove files that might have been left behind
sh runperfArchFilterRmCmd
sh "mkdir ${htmlPath}"
sh pythonDeployCmd
}
void deployDownstreamConfig(gitBranch) {
// This way we add downstream plugins and other configuration
dir('downstream_config') {
git branch: gitBranch, url: 'git://PATH_TO_YOUR_REPO_WITH_PIPELINES/runperf_config.git'
sh pythonDeployCmd
}
}
String getBkrInstallCmd(String hostBkrLinks, String hostBkrLinksFilter, String arch) {
// Constructs bash script to install pkgs from beaker
return ('\ndnf remove -y --skip-broken qemu-kvm;' +
'\nfor url in ' + hostBkrLinks + '; do dnf install -y --allowerasing --skip-broken ' +
'$(curl -k \$url | grep -oP \'href="\\K[^"]*(noarch|' + arch + ')\\.rpm\' | ' +
'sed -e "/^http/! s#^#$url/#" | grep -v $(for expr in ' + hostBkrLinksFilter + '; do ' +
'echo -n " -e $expr"; done)); done')
}
List getLatestDistros(String name, Integer limit, String arch) {
// Return latest $limit distros matching the name (use % to match anything)
distros = sh(returnStdout: true,
script: ('echo -n $(bkr distro-trees-list --arch ' + arch + ' --name=' + name +
' --limit ' + limit + ' --labcontroller ' + bkrLabController +
' --format json | grep \'"distro_name"\' | cut -d\'"\' -f4)'
)).trim().split()
return(distros)
}
@NonCPS
List getTestedDistros(String jobName, String distro) {
// Turn our distro (RHEL-8.0.0-20000000.n.0) into regex (RHEL-d.d.d-dddddddd.n.d)
// (this is unsafe method that leaves the '.' and such, but should do for now)
reNum = '[0-9]'
reDistro = distro.replaceAll(reNum, reNum)
reDistro = reDistro.replaceAll('%', '[^ ]*')
distros = []
for (build in Jenkins.instance.getJob(jobName).builds) {
build?.description?.eachMatch(reDistro) {
dist -> distros.add(dist)
}
}
return(distros)
}
String getLatestUntestedDistro(String distro, String arch) {
// Return latest distro that has not been tested by this job yet
tested_distros = getTestedDistros(env.JOB_NAME, distro)
latest_distros = getLatestDistros(distro, 10, arch)
for (dist in latest_distros) {
if (!(dist in tested_distros)) {
return(dist)
}
}
failBuild('No untested distros to try',
"All past 10 distros were already tested ${latest_distros}")
return("")
}
List getDistroRange(String[] range, String workerNode, String arch) {
// Wrapper to allow "..".split() as well as ["foo", "bar"]]
return(getDistroRange(range.toList(), workerNode, arch))
}
List getDistroRange(List range, String workerNode, String arch) {
// Find all distros between range[0] and range[1] revision (max 100 versions)
first = range[0]
last = range[1]
common = ''
for (i = 0; i < Math.min(first.length(), last.length()); i++) {
if (first[i] != last[i]) {
break
}
common += first[i]
}
if (first.contains('n') && last.contains('n')) {
common += '%n'
} else if (first.contains('d') && last.contains('d')) {
common += '%d'
}
node(workerNode) {
distros = getLatestDistros(common + '%', 100, arch).reverse();
distroRange = [];
i = 0;
while (i < distros.size()) {
if (distros[i] == last) {
break;
}
++i;
}
while (i < distros.size()) {
distroRange.add(distros[i]);
if (distros[i++] == first) {
break;
}
}
}
return(distroRange)
}
List getDistrosRange(String[] range, String workerNode, String arch) {
// Wrapper to allow "..".split() as well as ["foo", "bar"]]
return(getDistrosRange(range.toList(), workerNode, arch))
}
List getDistrosRange(List distrosRaw, String workerNode, String arch) {
// Process list of distros and replace '..' ranges with individual versions
println("getDistrosRange ${distrosRaw}")
List distros = []
for (distro in distrosRaw) {
if (distro.contains('..')) {
distroRange = getDistroRange(distro.split('\\.\\.'), workerNode, arch)
println("range ${distroRange}")
distros += distroRange.toList()
} else {
println("add ${distro}")
distros.add(distro)
}
}
return(distros)
}
@NonCPS
List getGoodBuildNumbers(String jobName) {
// Build is non-serializable object, we have to use NonCPS
// on the other hand we can not use copyArtifacts inside NonCPS
// therefore we have to only query for all descriptions and
// then iterate throught them, because we don't know how many
// builds we are going to need (copyArtifacts can fail)
builds = []
for (build in Jenkins.instance.getJob(jobName).builds) {
if (build?.description?.startsWith('BAD')) {
println("skip ${build.description} ${build.number}")
} else {
builds.add(build.number)
}
}
return builds
}
String setupScript(output, kernelArgs, bkrLinks, bkrFilter, arch, fioNbdSetup) {
// Generate additional parts of the setup script
if (kernelArgs) {
output += "\ngrubby --args '${kernelArgs}' --update-kernel=\$(grubby --default-kernel)"
}
// Ugly way of installing all arch's rpms from a site, allowing a filter
// this is usually used on koji/brew to allow updating certain packages
// warning: It does not work when the url rpm is older.
if (bkrLinks) {
output += getBkrInstallCmd(bkrLinks, bkrFilter, arch)
}
// Install deps and compile custom fio with nbd ioengine
if (fioNbdSetup) {
output += fioNbdScript
}
output += '\n'
return output
}
@NonCPS
def triggerRunperf(String jobName, Boolean waitForStart, String distro, String guestDistro,
String machine, String arch, String tests, String profiles, String srcBuild,
String hostKernelArgs, String hostBkrLinks, String hostBkrLinksFilter,
String guestKernelArgs, String guestBkrLinks, String guestBkrLinksFilter,
String upstreamQemuCommit, String descriptionPrefix,
Boolean pbenchPublish, Boolean fioNbdSetup, String noReferenceBuilds,
String cmpModelJob, String cmpModelBuild, String cmpTolerance,
String cmpStddevTolerance, String githubPublisherProject, String hostScript,
String workerScript) {
// Trigger a run-perf job setting all of the params according to arguments
// on waitForStart returns the triggered build.id, otherwise it
// returns srcBuild value
parameters = [
new StringParameterValue('DISTRO', distro),
new StringParameterValue('GUEST_DISTRO', guestDistro),
new StringParameterValue('MACHINE', machine),
new StringParameterValue('ARCH', arch),
new StringParameterValue('TESTS', tests),
new StringParameterValue('PROFILES', profiles),
new StringParameterValue('SRC_BUILD', srcBuild),
new StringParameterValue('HOST_KERNEL_ARGS', hostKernelArgs),
new StringParameterValue('HOST_BKR_LINKS', hostBkrLinks),
new StringParameterValue('HOST_BRK_LINKS_FILTER', hostBkrLinksFilter),
new StringParameterValue('GUEST_KERNEL_ARGS', guestKernelArgs),
new StringParameterValue('GUEST_BKR_LINKS', guestBkrLinks),
new StringParameterValue('GUEST_BKR_LINKS_FILTER', guestBkrLinksFilter),
new StringParameterValue('UPSTREAM_QEMU_COMMIT', upstreamQemuCommit),
new StringParameterValue('DESCRIPTION_PREFIX', descriptionPrefix),
new BooleanParameterValue('PBENCH_PUBLISH', pbenchPublish),
new BooleanParameterValue('FIO_NBD_SETUP', fioNbdSetup),
new StringParameterValue('NO_REFERENCE_BUILDS', noReferenceBuilds),
new StringParameterValue('CMP_MODEL_JOB', cmpModelJob),
new StringParameterValue('CMP_MODEL_BUILD', cmpModelBuild),
new StringParameterValue('CMP_TOLERANCE', cmpTolerance),
new StringParameterValue('CMP_STDDEV_TOLERANCE', cmpStddevTolerance),
new StringParameterValue('GITHUB_PUBLISHER_PROJECT', githubPublisherProject),
new TextParameterValue('HOST_SCRIPT', hostScript),
new TextParameterValue('WORKER_SCRIPT', workerScript)
]
job = Hudson.instance.getJob(jobName)
queue = job.scheduleBuild2(0, new ParametersAction(parameters))
if (waitForStart) {
println('Waiting for build to be scheduled to obtain srcBuild ID')
build = queue.waitForStart()
srcBuild = "${build.id}"
}
// Explicitly clean build, job and queue, otherwise we get CPS failures
build = job = queue = null
return(srcBuild)
}
@NonCPS
void tryOtherDistros(String rawDistro, String arch) {
// Re-trigger the job with another untested distro if possible
String strProvisionFail = 'Provisioning failed'
if (!rawDistro.startsWith('latest-')) {
// Using strict distro version
failBuild(strProvisionFail,
"Provisioning failed, bailing out as we are using strict distro ${rawDistro}")
}
if (rawDistro.startsWith('latest-untested-')) {
latestDistro = rawDistro
} else {
latestDistro = "latest-untested-${rawDistro[7..-1]}"
}
triggerRunperf(env.JOB_NAME, false, latestDistro, params.GUEST_DISTRO, params.MACHINE, params.ARCH,
params.TESTS, params.PROFILES, params.SRC_BUILD, params.HOST_KERNEL_ARGS,
params.HOST_BKR_LINKS, params.HOST_BRK_LINKS_FILTER, params.GUEST_KERNEL_ARGS,
params.GUEST_BKR_LINKS, params.GUEST_BKR_LINKS_FILTER, params.UPSTREAM_QEMU_COMMIT,
params.DESCRIPTION_PREFIX, params.PBENCH_PUBLISH, params.FIO_NBD_SETUP,
params.NO_REFERENCE_BUILDS, params.CMP_MODEL_JOB, params.CMP_MODEL_BUILD,
params.CMP_TOLERANCE, params.CMP_STDDEV_TOLERANCE, params.GITHUB_PUBLISHER_PROJECT,
params.HOST_SCRIPT, params.WORKER_SCRIPT)
}
@NonCPS
def getBuildEnv(String jobName, String buildName) {
env = Hudson.instance.getJob(jobName).getBuildByNumber(buildName as int).getEnvironment()
str_env = "${env}"
env = null
return str_env
}
into $YOUR_SHARED_LIB_REPO/vars/runperf.groovy and you
should be good to go. Feel free to ping me if anything
goes wrong.