https://gitlab.opengeosys.org/ogs/ogs.git
Tip revision: a102c0e089fce290d3d907518b7e59556e1a7b22 authored by Dmitri Naumov on 03 January 2020, 18:44:44 UTC
Changelog references update.
Changelog references update.
Tip revision: a102c0e
Jenkinsfile
#!/usr/bin/env groovy
@Library('jenkins-pipeline@1.0.22') _
def stage_required = [build: false, full: false]
def build_shared = 'ON'
pipeline {
agent none
options {
ansiColor('xterm')
timestamps()
buildDiscarder(logRotator(numToKeepStr: '30', artifactNumToKeepStr: '10'))
timeout(time: 6, unit: 'HOURS')
}
parameters {
booleanParam(name: 'docker_conan', defaultValue: true)
booleanParam(name: 'docker_conan_debug', defaultValue: true)
booleanParam(name: 'docker_conan_gui', defaultValue: true)
booleanParam(name: 'eve_serial', defaultValue: true)
booleanParam(name: 'eve_parallel', defaultValue: true)
booleanParam(name: 'win', defaultValue: true)
booleanParam(name: 'mac', defaultValue: true)
booleanParam(name: 'clang_analyzer', defaultValue: true)
booleanParam(name: 'master_jobs', defaultValue: true)
}
stages {
// *************************** Git Check **********************************
stage('Pre-checks') {
agent { label "master"}
steps {
sh "pre-commit install"
sh "pre-commit run --all-files"
sh "git config core.whitespace -blank-at-eof"
sh "git diff --check `git merge-base origin/master HEAD` HEAD -- . ':!*.md' ':!*.pandoc' ':!*.asc' ':!*.dat' ':!*.ts'"
dir('scripts/jenkins') { stash(name: 'known_hosts', includes: 'known_hosts') }
ciSkip action: 'check' // Check for [ci skip] or [web] commit message.
// ********* Check changesets for conditional stage execution **********
script {
def causes = currentBuild.getBuildCauses()
for(cause in causes) {
if (cause.class.toString().contains("UserIdCause")) {
echo "Doing full build because job was started by user."
stage_required.full = true
env.CI_SKIP = "false"
return true
}
}
if (env.JOB_NAME == 'ufz/ogs/master') {
build_shared = 'OFF'
}
if (currentBuild.number == 1 || buildingTag()) {
stage_required.full = true
return true
}
def changeLogSets = currentBuild.changeSets
for (int i = 0; i < changeLogSets.size(); i++) {
def entries = changeLogSets[i].items
for (int j = 0; j < entries.length; j++) {
def paths = new ArrayList(entries[j].affectedPaths)
for (int p = 0; p < paths.size(); p++) {
def path = paths[p]
if (path.matches("Jenkinsfile")) {
stage_required.full = true
echo "Doing full build."
return true
}
if (path.matches("^(CMakeLists.txt|scripts|Applications|BaseLib|FileIO|GeoLib|MaterialLib|MathLib|MeshGeoToolsLib|MeshLib|NumLib|ProcessLib|SimpleTests|Tests).*")
&& !stage_required.build) {
stage_required.build = true
echo "Doing regular build."
}
}
}
}
if(!(stage_required.build || stage_required.full)) {
currentBuild.result='NOT_BUILT'
}
}
}
// Mark build as NOT_BUILT when [ci skip] commit message was found
post { always { ciSkip action: 'postProcess' } }
}
stage('Build') {
parallel {
// ************************ Docker-Conan *******************************
stage('Docker-Conan') {
when {
beforeAgent true
expression { return params.docker_conan && (stage_required.build || stage_required.full) }
}
agent {
dockerfile {
filename 'Dockerfile.gcc.full'
dir 'scripts/docker'
label 'docker'
args '-v /home/jenkins/cache/ccache:/opt/ccache -v /home/jenkins/cache/conan/.conan:/opt/conan/.conan'
additionalBuildArgs '--pull'
}
}
environment {
OMP_NUM_THREADS = '1'
LD_LIBRARY_PATH = "$WORKSPACE/build/lib"
}
steps {
script {
sh 'git submodule sync'
configure {
cmakeOptions =
"-DBUILD_SHARED_LIBS=OFF " +
'-DOGS_CPU_ARCHITECTURE=generic ' +
'-DOGS_BUILD_UTILS=ON ' +
'-DOGS_CONAN_BUILD=missing ' +
'-DOGS_USE_CVODE=ON ' +
'-DOGS_USE_MFRONT=ON ' +
'-DOGS_USE_PYTHON=ON '
}
build {
target="package"
log="build1.log"
}
build { target="tests" }
build { target="ctest" }
build { target="doc" }
}
}
post {
always {
xunit([
// Testing/-folder is a CTest convention
CTest(pattern: 'build/Testing/**/*.xml'),
GoogleTest(pattern: 'build/Tests/testrunner.xml')
])
recordIssues enabledForFailure: true, filters: [
excludeFile('.*qrc_icons\\.cpp.*'), excludeFile('.*QVTKWidget.*'),
excludeFile('.*\\.conan.*/TFEL/.*'),
excludeFile('.*ThirdParty/MGIS/src.*'),
excludeFile('.*MaterialLib/SolidModels/MFront/.*\\.mfront'),
excludeFile('.*MaterialLib/SolidModels/MFront/.*\\.hxx'),
excludeMessage('.*tmpnam.*')],
tools: [gcc4(name: 'GCC', pattern: 'build/build*.log')],
unstableTotalAll: 1
recordIssues enabledForFailure: true, filters: [
excludeFile('-'), excludeFile('.*Functional\\.h'),
excludeFile('.*gmock-.*\\.h'), excludeFile('.*gtest-.*\\.h')
],
// Doxygen is handled by gcc4 parser as well
tools: [gcc4(name: 'Doxygen', id: 'doxygen',
pattern: 'build/DoxygenWarnings.log')],
unstableTotalAll: 1
}
success {
publishHTML(target: [allowMissing: false, alwaysLinkToLastBuild: true,
keepAll: true, reportDir: 'build/docs', reportFiles: 'index.html',
reportName: 'Doxygen'])
archiveArtifacts 'build/*.tar.gz,build/conaninfo.txt'
script {
if (env.JOB_NAME == 'ufz/ogs/master') {
// Deploy Doxygen
unstash 'known_hosts'
sshagent(credentials: ['www-data_jenkins']) {
sh 'rsync -a --delete --stats -e "ssh -o UserKnownHostsFile=' +
'known_hosts" build/docs/. ' +
'www-data@jenkins.opengeosys.org:/var/www/doxygen.opengeosys.org'
}
}
}
}
}
}
// *********************** Docker-Conan-GUI *****************************
stage('Docker-Conan-GUI') {
when {
beforeAgent true
expression { return params.docker_conan_gui && (stage_required.build || stage_required.full) }
}
agent {
dockerfile {
filename 'Dockerfile.gcc.gui'
dir 'scripts/docker'
label 'docker'
args '-v /home/jenkins/cache/ccache:/opt/ccache -v /home/jenkins/cache/conan/.conan:/opt/conan/.conan'
additionalBuildArgs '--pull'
}
}
steps {
script {
sh 'git submodule sync'
sh "conan remove --system-reqs '*'"
configure {
cmakeOptions =
"-DBUILD_SHARED_LIBS=${build_shared} " +
'-DOGS_CPU_ARCHITECTURE=generic ' +
'-DOGS_USE_PCH=OFF ' + // see #1992
'-DOGS_BUILD_GUI=ON ' +
'-DOGS_BUILD_UTILS=ON ' +
'-DOGS_CONAN_BUILD=missing ' +
'-DOGS_BUILD_TESTS=OFF ' +
'-DOGS_USE_NETCDF=ON '
}
build {
target="package"
log="build1.log"
}
configure { // CLI + GUI + Python
cmakeOptions = '-DOGS_USE_PYTHON=ON '
keepDir = true
}
build {
target="package"
log="build2.log"
}
build { target="cppcheck" }
}
}
post {
always {
recordIssues enabledForFailure: true, filters: [
excludeFile('.*qrc_icons\\.cpp.*'),
excludeFile('ncGroup\\.h'),
excludeMessage('.*tmpnam.*')],
tools: [gcc4(name: 'GCC-GUI', id: 'gcc4-gui',
pattern: 'build/build*.log')],
unstableTotalAll: 1
recordIssues enabledForFailure: true,
tools: [cppCheck(pattern: 'build/cppcheck.log')]
}
success { archiveArtifacts 'build/*.tar.gz,build/conaninfo.txt,build/cppcheck.log' }
}
}
// ********************* Docker-Conan-Debug ****************************
stage('Docker-Conan-Debug') {
when {
beforeAgent true
expression { return params.docker_conan_debug && (stage_required.build || stage_required.full) }
}
agent {
dockerfile {
filename 'Dockerfile.gcc.full'
dir 'scripts/docker'
label 'docker'
args '-v /home/jenkins/cache/ccache:/opt/ccache -v /home/jenkins/cache/conan/.conan:/opt/conan/.conan'
additionalBuildArgs '--pull'
}
}
steps {
script {
sh 'git submodule sync'
configure {
cmakeOptions =
"-DBUILD_SHARED_LIBS=${build_shared} " +
'-DOGS_CONAN_BUILD=missing ' +
'-DOGS_CONAN_BUILD_TYPE=Release ' +
'-DOGS_CPU_ARCHITECTURE=generic '
config = 'Debug'
}
build { }
build { target = 'tests' }
}
}
post {
always {
xunit([GoogleTest(pattern: 'build/Tests/testrunner.xml')])
}
}
}
// **************************** eve ************************************
stage('Frontend2 (serial)') {
when {
beforeAgent true
expression { return params.eve_serial && (stage_required.build || stage_required.full) }
}
agent { label "frontend2"}
environment {
OMP_NUM_THREADS = '1'
}
steps {
script {
sh 'git submodule sync'
configure {
cmakeOptions =
'-DOGS_BUILD_UTILS=ON ' +
'-DBUILD_SHARED_LIBS=ON ' +
'-DCMAKE_INSTALL_PREFIX=/global/apps/ogs/head/standard ' +
'-DOGS_MODULEFILE=/global/apps/modulefiles/ogs/head/standard '
env = 'eve/cli.sh'
}
build {
env = 'eve/cli.sh'
cmd_args = '-l 30'
}
build {
env = 'eve/cli.sh'
target = 'tests'
}
build {
env = 'eve/cli.sh'
target = 'ctest'
}
}
}
post {
always {
xunit([
CTest(pattern: 'build/Testing/**/*.xml'),
GoogleTest(pattern: 'build/Tests/testrunner.xml')
])
}
success {
script {
if (env.JOB_NAME == 'ufz/ogs/master') {
sh 'rm -rf /global/apps/ogs/head/standard'
build {
env = 'eve/cli.sh'
target = 'install'
}
}
}
}
}
}
stage('Frontend2 (parallel)') {
when {
beforeAgent true
expression { return params.eve_parallel && (stage_required.build || stage_required.full) }
}
agent { label "frontend2"}
environment {
OMP_NUM_THREADS = '1'
}
steps {
script {
sh 'git submodule sync'
configure {
cmakeOptions =
'-DOGS_USE_PETSC=ON ' +
'-DBUILD_SHARED_LIBS=ON ' +
'-DCMAKE_INSTALL_PREFIX=/global/apps/ogs/head/petsc ' +
'-DOGS_MODULEFILE=/global/apps/modulefiles/ogs/head/petsc '
env = 'eve/petsc.sh'
}
build {
env = 'eve/petsc.sh'
cmd_args = '-l 30'
}
build {
env = 'eve/petsc.sh'
target = 'tests'
}
build {
env = 'eve/petsc.sh'
target = 'ctest'
}
}
}
post {
always {
xunit([
CTest(pattern: 'build/Testing/**/*.xml'),
GoogleTest(pattern: 'build/Tests/testrunner.xml')
])
}
success {
script {
if (env.JOB_NAME == 'ufz/ogs/master') {
sh 'rm -rf /global/apps/ogs/head/petsc'
build {
env = 'eve/petsc.sh'
target = 'install'
}
}
}
}
}
}
// ************************** Windows **********************************
stage('Win') {
when {
beforeAgent true
expression { return params.win && (stage_required.build || stage_required.full) }
}
agent {label 'win && conan' }
environment {
MSVC_NUMBER = '15'
MSVC_VERSION = '2017'
OMP_NUM_THREADS = '1'
CC = 'clcache'
CXX = 'clcache'
}
steps {
script {
def num_threads = env.NUM_THREADS
bat 'git submodule sync'
bat 'conan remove --locks'
configure { // CLI + GUI
cmakeOptions =
"-DBUILD_SHARED_LIBS=OFF " +
'-DOGS_DOWNLOAD_ADDITIONAL_CONTENT=ON ' +
'-DOGS_BUILD_GUI=ON ' +
'-DOGS_BUILD_UTILS=ON ' +
'-DOGS_CONAN_BUILD=missing ' +
'-DOGS_BUILD_SWMM=ON ' +
'-DOGS_USE_NETCDF=ON '
}
build {
target="package"
log="build1.log"
cmd_args="-l ${num_threads}"
}
configure { // CLI + GUI + Python
cmakeOptions = '-DOGS_USE_PYTHON=ON '
keepDir = true
}
build {
target="package"
log="build2.log"
}
build { target="tests" }
build { target="ctest" }
}
}
post {
always {
xunit([
CTest(pattern: 'build/Testing/**/*.xml'),
GoogleTest(pattern: 'build/Tests/testrunner.xml')
])
recordIssues enabledForFailure: true, filters: [
excludeFile('.*\\.conan.*'), excludeFile('.*ThirdParty.*'),
excludeFile('.*thread.hpp')],
tools: [msBuild(name: 'MSVC', pattern: 'build/build*.log')],
qualityGates: [[threshold: 10, type: 'TOTAL', unstable: true]]
}
success {
archiveArtifacts 'build/*.zip,build/conaninfo.txt'
}
}
}
// ****************************** Mac **********************************
stage('Mac') {
when {
beforeAgent true
expression { return params.mac && (stage_required.build || stage_required.full) }
}
agent { label "mac"}
environment {
OMP_NUM_THREADS = '1'
}
steps {
script {
sh 'git submodule sync'
configure {
cmakeOptions =
"-DBUILD_SHARED_LIBS=${build_shared} " +
'-DOGS_CPU_ARCHITECTURE=core2 ' +
'-DOGS_DOWNLOAD_ADDITIONAL_CONTENT=ON ' +
'-DOGS_BUILD_UTILS=ON ' +
'-DOGS_CONAN_BUILD=missing ' +
'-DCMAKE_OSX_DEPLOYMENT_TARGET="10.14" ' +
'-DOGS_USE_NETCDF=ON '
}
build {
target="package"
log = "build.log"
}
build { target = 'tests' }
build { target = 'ctest' }
}
}
post {
always {
xunit([
CTest(pattern: 'build/Testing/**/*.xml'),
GoogleTest(pattern: 'build/Tests/testrunner.xml')
])
recordIssues enabledForFailure: true, filters: [
excludeFile('.*qrc_icons\\.cpp.*'), excludeMessage('.*QVTKWidget.*'),
excludeMessage('.*tmpnam.*')],
tools: [clang(name: 'Clang (macOS)', pattern: 'build/build.log',
id: 'clang-mac')], unstableTotalAll: 3
}
success {
archiveArtifacts 'build/*.tar.gz,build/*.dmg,build/conaninfo.txt'
}
}
}
// ************************* Clang-Analyzer *********************************
stage('Clang-Analyzer') {
when {
beforeAgent true
expression { return params.clang_analyzer && (stage_required.build || stage_required.full) }
}
agent {
dockerfile {
filename 'Dockerfile.clang.full'
dir 'scripts/docker'
label 'docker'
args '-v /home/jenkins/cache/ccache:/opt/ccache -v /home/jenkins/cache/conan/.conan:/opt/conan/.conan'
additionalBuildArgs '--pull'
}
}
steps {
script {
sh 'git submodule sync'
sh 'find $CONAN_USER_HOME -name "system_reqs.txt" -exec rm {} \\;'
configure {
cmakeOptions =
"-DBUILD_SHARED_LIBS=${build_shared} " +
'-DBUILD_TESTING=OFF ' +
'-DCMAKE_CXX_CLANG_TIDY=clang-tidy-8 '
}
build { log = 'build.log' }
}
}
post {
always {
recordIssues enabledForFailure: true, filters: [
excludeFile('.*\\.conan.*')],
tools: [clangTidy(name: 'Clang-Tidy', pattern: 'build/build.log')],
qualityGates: [[threshold: 247, type: 'TOTAL', unstable: true]]
}
}
}
// ************************* Check headers *********************************
stage('Check headers') {
when {
beforeAgent true
allOf {
expression { return params.master_jobs && (stage_required.build || stage_required.full) }
environment name: 'JOB_NAME', value: 'ufz/ogs/master'
}
}
agent {
dockerfile {
filename 'Dockerfile.clang.full'
dir 'scripts/docker'
label 'docker'
args '-v /home/jenkins/cache/ccache:/opt/ccache -v /home/jenkins/cache/conan/.conan:/opt/conan/.conan'
additionalBuildArgs '--pull'
}
}
steps {
script {
sh 'git submodule sync'
sh 'find $CONAN_USER_HOME -name "system_reqs.txt" -exec rm {} \\;'
try {
configure {
cmakeOptions = '-DOGS_CHECK_HEADER_COMPILATION=ON'
dir = 'build-check-header'
}
}
catch(err) {
sh 'cat build-check-header/CMakeFiles/CMakeError.log'
unstable('check-header failed!')
}
}
}
}
// ************************* Tests-Large *******************************
stage('Tests-Large') {
when {
beforeAgent true
allOf {
expression { return params.master_jobs && (stage_required.build || stage_required.full) }
environment name: 'JOB_NAME', value: 'ufz/ogs/master'
}
}
agent {
dockerfile {
filename 'Dockerfile.gcc.full'
dir 'scripts/docker'
label 'singularity1 || envinf1'
args '-v /home/jenkins/cache/ccache:/opt/ccache -v /home/jenkins/cache/conan/.conan:/opt/conan/.conan'
additionalBuildArgs '--pull'
}
}
environment {
OMP_NUM_THREADS = '1'
}
steps {
script {
configure { }
build { target = 'ctest-large' }
}
}
post {
always {
xunit([CTest(pattern: 'build/Testing/**/*.xml')])
dir('build') { deleteDir() }
}
}
}
// ************************** Sanitizer ********************************
stage('Sanitizer') {
when {
beforeAgent true
allOf {
expression { return params.master_jobs && (stage_required.build || stage_required.full)}
environment name: 'JOB_NAME', value: 'ufz/ogs/master'
}
}
agent {
dockerfile {
filename 'Dockerfile.clang.full'
dir 'scripts/docker'
label 'docker'
args '-v /home/jenkins/cache/ccache:/opt/ccache -v /home/jenkins/cache/conan/.conan:/opt/conan/.conan'
additionalBuildArgs '--pull'
}
}
environment {
UBSAN_OPTIONS = 'print_stacktrace=1'
LSAN_OPTIONS = "suppressions=$WORKSPACE/scripts/test/leak_sanitizer.suppressions"
}
steps {
script {
sh 'git submodule sync'
sh 'find $CONAN_USER_HOME -name "system_reqs.txt" -exec rm {} \\;'
configure {
cmakeOptions =
'-DOGS_ADDRESS_SANITIZER=ON ' +
'-DOGS_UNDEFINED_BEHAVIOR_SANITIZER=ON ' +
'-DOGS_BUILD_UTILS=ON '
}
try {
build { target = 'tests' }
}
catch(err) { echo "Clang sanitizer for unit tests failed!" }
try {
build { target = 'ctest' }
}
catch(err) { echo "Clang sanitizer for end-to-end tests failed!" }
}
}
// Currently disabled because of Java out ouf heap space errors
// post {
// always {
// recordIssues enabledForFailure : true,
// filters: [includeCategory('clang-analyzer.*')],
// tools: [clang(name: 'Clang (StaticAnalyzer)')]
// }
// }
}
// ********************* Update ufz/ogs-data ***************************
stage('Update ogs-data') {
when {
beforeAgent true
allOf {
expression { return params.master_jobs }
environment name: 'JOB_NAME', value: 'ufz/ogs/master'
}
}
agent { label "master"}
steps {
script {
dir('ogs') { checkout scm }
dir('ogs-data') {
checkout(changelog: false, poll: false, scm: [$class: 'GitSCM',
extensions: [[$class: 'CloneOption', shallow: true]],
userRemoteConfigs: [[
credentialsId: '2719b702-1298-4e87-8464-5dfc62fbd923',
url: 'https://github.com/ufz/ogs-data']]])
sh 'rsync -av --delete --exclude .git/ ../ogs/Tests/Data/ .'
sh "git add --all . && git diff --quiet && git diff --staged --quiet || git commit -am 'Update'"
withCredentials([usernamePassword(
credentialsId: '2719b702-1298-4e87-8464-5dfc62fbd923',
passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME')]) {
sh 'git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/ufz/ogs-data HEAD:master'
}
}
}
}
}
// ******************** Container Maker Images *************************
stage('Container Maker Images') {
when {
beforeAgent true
allOf {
expression { return params.master_jobs }
environment name: 'JOB_NAME', value: 'ufz/ogs/master'
}
}
agent { label 'docker'}
steps {
script {
sh '''git submodule update --init ThirdParty/container-maker
virtualenv .venv
source .venv/bin/activate
pip install -r ThirdParty/container-maker/requirements.txt
export PYTHONPATH="${PYTHONPATH}:${PWD}/ThirdParty/container-maker"
python ThirdParty/container-maker/ogscm/cli.py -B -C -R --ogs . --pm system --cvode --ompi off 2.1.6 3.1.4 4.0.1
'''.stripIndent()
}
}
post {
success {
archiveArtifacts('_out/images/*.sif')
dir('_out') { deleteDir() } // Cleanup
}
}
}
// ********************* Push Docker Images ***************************
stage('Push Docker Images') {
when {
beforeAgent true
allOf {
expression { return params.master_jobs }
environment name: 'JOB_NAME', value: 'ufz/ogs/master'
}
}
agent { label 'docker'}
steps {
script {
dir('scripts/docker') {
def gccImage = docker.build("ogs6/gcc:latest", "-f Dockerfile.gcc.full .")
def gccGuiImage = docker.build("ogs6/gcc:gui", "-f Dockerfile.gcc.gui .")
def clangImage = docker.build("ogs6/clang:latest", "-f Dockerfile.clang.full .")
withCredentials([usernamePassword(credentialsId: 'docker-hub-credentials',
passwordVariable: 'pw', usernameVariable: 'docker_user')]) {
sh 'echo $pw | docker login -u $docker_user --password-stdin'
retry(3) {
gccImage.push()
gccGuiImage.push()
clangImage.push()
}
}
}
}
}
}
// *************************** Web *************************************
stage('Web') {
agent {
dockerfile {
filename 'Dockerfile.web'
dir 'scripts/docker'
label 'docker'
}
}
when {
beforeAgent true
environment name: 'JOB_NAME', value: 'ufz/ogs/master'
}
steps {
dir('web') {
sh 'yarn'
sh 'hugo'
sh 'broken-links-checker --path ./public --baseUrl https://www.opengeosys.org'
}
}
}
// *************************** Post ************************************
stage('Post') {
agent { label "master"}
when { buildingTag() }
steps {
script {
currentBuild.keepLog(true)
currentBuild.displayName = tag
}
}
}
} // end parallel
} // end stage Build
}
}