jenkins-pipelines/pipelines/monolithic.Jenkinsfile

// vim: syn=groovy

//
// Copyright (c) 2022 Wind River Systems, Inc.
//
// SPDX-License-Identifier: Apache-2.0
//

library "common@${params.JENKINS_SCRIPTS_BRANCH}"

PROPS = null
IMG_PARAMS = null
IMAGES_FAILED = false

def parseProps(text) {
    def x = {}
    for (line in text.split (/\n+/)) {
        if (line.matches (/\s*(?:#.*)?#/)) {
            continue
        }
        parts = line.split ("=", 2)
        key = parts[0]
        value = parts[1]
        x."${key}" = value
    }
    return x
}

def loadEnv() {
    def data = {}
    data.NEED_BUILD = false
    data.SUPPRESS_DOCKER_IMAGE_BUILD_ERRORS = params.SUPPRESS_DOCKER_IMAGE_BUILD_ERRORS
    ws(params.BUILD_HOME) {
        if (fileExists ("NEED_BUILD")) {
            data.NEED_BUILD = true
        }
    }
    final String configText = sh (script: "${Constants.SCRIPTS_DIR}/print-config.sh", returnStdout: true)
    final props = parseProps (configText)
    data.BUILD_OUTPUT_HOME_URL = props.BUILD_OUTPUT_HOME_URL
    data.PUBLISH_URL = props.PUBLISH_URL
    PROPS = data
    return data.NEED_BUILD
}

def partJobName (name) {
    final String folder = env.JOB_NAME.replaceAll (/(.*\/).+$/, '$1');
    if (folder == env.JOB_NAME) {
        error "This job must be in a Jenkins folder!"
    }
    return "/" + folder + "parts/" + name
}

def runPart (name, params = [], propagate = true) {
    // Tell Jenkins to checkout the same commit of the sub-job's Jenkinsfile,
    // as the current builds' Jenkinsfile's commit.
    final gitRef = string (name: 'JENKINS_SCRIPTS_BRANCH', value: env.GIT_COMMIT)
    return build (
        job: partJobName (name),
        parameters: copyCurrentParams() + [ gitRef ] + params,
        propagate: propagate
    )
}

def runImagesPart (name, params = []) {
    // Ignore docker image - related errors. In this case We
    // prevent sub-jobs from raising exceptions and failing the
    // current build. Instead we note when an image-related
    // job has failed, and skip all subsequent image-related
    // jobs.
    if (PROPS.SUPPRESS_DOCKER_IMAGE_BUILD_ERRORS) {
        if (!IMAGES_FAILED) {
            final jobName = partJobName (name)
            final res = runPart (name, IMG_PARAMS + params, false).result
            if (res == 'ABORTED') {
                // FIXME: make current build ABORTED here
                error ("child job ${jobName} aborted")
            }
            if (res == 'SUCCESS' || res == 'UNSTABLE') {
                return true
            }
            print ("*** ERROR: child job ${jobName} failed!")
            IMAGES_FAILED = true
        }
        return false
    }
    // Otherwise, just call the subjob normally - ie its failure
    // will propagate to the current build
    runPart (name, IMG_PARAMS + params)
    return true
}

def printBuildFooter() {
    if (PROPS) {
        String msg = ""
        msg += "\n"
        msg += "========================================\n"
        msg += "\n"
        if (PROPS.NEED_BUILD) {
            msg += "Build output:   ${PROPS.BUILD_OUTPUT_HOME_URL}\n"
            if (PROPS.PUBLISH_URL) {
                msg += "Publish output: ${PROPS.PUBLISH_URL}\n"
            }
            if (IMAGES_FAILED) {
                msg += "\n"
                msg += "WARNING:\n"
                msg += "WARNING: docker images build attempted, but failed!\n"
                msg += "WARNING: see log output above\n"
                msg += "WARNING:\n"
            }
        }
        else {
            echo "*** NO CHANGES - BUILD NOT REQUIRED"
        }
        msg += "\n"
        msg += "========================================\n"
        msg += "\n"
        echo (msg)
    }
}

setBuildDescr()

pipeline {
    agent any
    options {
        timestamps()
    }
    parameters {
        string (
            name: 'MASTER_JOB_NAME'
        )
        string (
            name: 'MASTER_BUILD_NUMBER'
        )
        string (
            name: 'BUILD_HOME'
        )
        string (
            name: 'TIMESTAMP',
        )
        string (
            name: 'PUBLISH_TIMESTAMP'
        )
        booleanParam (
            name: 'REBUILD_BUILDER_IMAGES'
        )
        booleanParam (
            name: 'BUILDER_USE_DOCKER_CACHE'
        )
        booleanParam (
            name: 'REFRESH_SOURCE'
        )
        booleanParam (
            name: 'BUILD_PACKAGES'
        )
        string (
            name: 'BUILD_PACKAGES_LIST'
        )
        booleanParam (
            name: 'PKG_REUSE'
        )
        booleanParam (
            name: 'BUILD_ISO'
        )
        booleanParam (
            name: 'BUILD_RT'
        )
        booleanParam (
            name: 'DRY_RUN'
        )
        booleanParam (
            name: 'SHELL_XTRACE'
        )
        booleanParam (
            name: 'CLEAN_PACKAGES'
        )
        booleanParam (
            name: 'CLEAN_ISO'
        )
        booleanParam (
            name: 'CLEAN_REPOMGR'
        )
        booleanParam (
            name: 'CLEAN_DOWNLOADS'
        )
        booleanParam (
            name: 'CLEAN_DOCKER'
        )
        booleanParam (
            name: 'FORCE_BUILD'
        )
        booleanParam (
            name: 'BUILD_HELM_CHARTS'
        )
        booleanParam (
            name: 'FORCE_BUILD_WHEELS'
        )
        booleanParam (
            name: 'BUILD_DOCKER_BASE_IMAGE'
        )
        booleanParam (
            name: 'BUILD_DOCKER_IMAGES'
        )
        string (
            name: 'DOCKER_IMAGE_LIST'
        )
        booleanParam (
            name: 'PUSH_DOCKER_IMAGES'
        )
        booleanParam (
            name: 'SUPPRESS_DOCKER_IMAGE_BUILD_ERRORS',
            defaultValue: false
        )
        booleanParam (
            name: 'IMPORT_BUILD'
        )
        string (
            name: 'IMPORT_BUILD_DIR'
        )
        booleanParam (
            name: 'USE_DOCKER_CACHE',
        )
        string (
            name: 'JENKINS_SCRIPTS_BRANCH'
        )
        text (
            name: 'PATCH_LIST',
            defaultValue: '-',
            description: '''\
<pre><code>List of Gerrit URLs to apply before running the build, one per line "[PATH] URL REF", eg:

  https://review.opendev.org/starlingx/config     refs/changes/71/859571/4
  https://review.opendev.org/starlingx/stx-puppet refs/changes/75/859575/1
  https://review.opendev.org/starlingx/tools      refs/changes/76/859576/2

or with paths relative to repo root:

  cgcs-root/stx/config     https://review.opendev.org/starlingx/config     refs/changes/71/859571/4
  cgcs-root/stx/stx-puppet https://review.opendev.org/starlingx/stx-puppet refs/changes/75/859575/1
  stx-tools                https://review.opendev.org/starlingx/tools      refs/changes/76/859576/2
</code></pre>
            '''
        )

    }
    stages {
        stage('INIT') {
            steps {
                script {
                    // Initialize BUILD_HOME, create build.conf & stx.conf
                    runPart ("init-env")
                    // Update source tree
                    runPart ("clone-source")
                    // create BUILD & stx.conf
                    runPart ("configure-build")
                    // Stop containers before updating source treee
                    runPart ("stop-containers")
                    // Create chnagelog, LAST_COMMITS, NEED_BUILD etc
                    runPart ("create-changelog")
                    // Is build required?
                    if (loadEnv()) {
                        IMG_PARAMS = [ string (name: 'BUILD_STREAM', value: 'stable') ]
                    }
                    else {
                        println "*** NO CHANGES, BUILD NOT REQUIRED ***"
                    }
                }
            }
        }
        // This stage runs only if build is required
        stage('X0') {
            when { expression { PROPS.NEED_BUILD } }
            stages {
                stage('PREPARE') {
                    steps {
                        // Delete or keep packages, aptly state, etc depending on build params
                        runPart ("clean-build")
                        // start containers
                        runPart ("start-containers")
                        // login to docker early to catch login errors
                        runPart ("docker-login")
                    }
                }
                // populate mirror/
                stage('DOWNLOAD') {
                    steps {
                        runPart ("download-prerequisites")
                    }
                }
                // build packages
                stage('PACKAGES') {
                    when { expression { params.BUILD_PACKAGES } }
                    steps {
                        runPart ("build-packages")
                        runPart ("publish-packages")
                    }
                }
                // Generate initial helm charts. We will re-generate them after
                // building docker images, if requested in order to replace
                // image tags by locally-built images
                stage('HELM:initial') {
                    when { expression { params.BUILD_HELM_CHARTS } }
                    steps {
                        runPart ("build-helm-charts", IMG_PARAMS)
                        runPart ("publish-helm-charts", IMG_PARAMS)
                    }
                }
                // Build ISO & images in parallel
                stage('X1') { parallel {
                    stage('ISO') {
                        when { expression { params.BUILD_ISO } }
                        steps { script {
                            runPart ("build-iso")
                            runPart ("publish-iso")
                        } }
                    } // stage('ISO')
                    stage('IMAGES') {
                        when { expression { params.BUILD_DOCKER_BASE_IMAGE || params.BUILD_DOCKER_IMAGES } }
                        stages {
                            stage('IMAGES:base') {
                                when { expression { ! IMAGES_FAILED && params.BUILD_DOCKER_BASE_IMAGE } }
                                steps { script {
                                    runImagesPart ("build-docker-base")
                                } }
                            }
                            stage('IMAGES:wheels') {
                                when { expression { ! IMAGES_FAILED && params.BUILD_DOCKER_IMAGES } }
                                steps { script {
                                    runImagesPart ("build-wheels")
                                    runImagesPart ("publish-wheels")
                                } }
                            }
                            stage('IMAGES:images') {
                                when { expression { ! IMAGES_FAILED && params.BUILD_DOCKER_IMAGES } }
                                steps { script {
                                    runImagesPart ("build-docker-images")
                                } }
                            }
                            stage('IMAGES:helm') {
                                // Rebuild helm charts even if image builds failed.
                                // This will record any images that were built sucessfully in the helm charts
                                when { expression { params.BUILD_DOCKER_IMAGES && params.BUILD_HELM_CHARTS } }
                                steps { script {
                                    runPart ("build-helm-charts", IMG_PARAMS)
                                    runPart ("publish-helm-charts", IMG_PARAMS)
                                } }
                            }
                            stage('IMAGES:symlinks') {
                                // Create the symlink even if some images failed.
                                // FIXME: remove all logic re publishing failed docker builds
                                //        once all images have been fixed for Debian
                                // when { expression { ! IMAGES_FAILED } }
                                steps { script {
                                    // copy image lists to publish root and create the "latest_docker_image_build" symlinks
                                    // in publish and archive roots
                                    sh ("BUILD_STATUS=success ${Constants.SCRIPTS_DIR}/create-latest-containers-symlinks.sh")
                                } }
                            }
                        } // stages
                    } // stage('IMAGES')
                    stage('export-dir') { steps {
                        runPart ("build-export-dir")
                    } }
                } }// stage('X1')
            } // stages

            post {
                always {
                    echo "build result: ${currentBuild.result}"
                    runPart ("stop-containers")
                    runPart ("archive-misc") // archive anything we may have missed
                    saveCurrentJenkinsBuildInfo()   // save this job's build number on disk (for publish-logs)
                }
                success {
                    // copy LAST_COMMITS to archive root & update the "latest_build" symlink in
                    // both archive and publish roots
                    sh ("BUILD_STATUS=success ${Constants.SCRIPTS_DIR}/create-latest-symlinks.sh")
                    printBuildFooter() //  Print archive & publish URLs
                    runPart ("publish-logs")        // publish this job's Jenkins log
                }
                unsuccessful {
                    sh ("BUILD_STATUS=fail ${Constants.SCRIPTS_DIR}/create-latest-symlinks.sh")
                    runPart ("publish-logs")        // publish this job's Jenkins log
                }
            }
        } // stage X0
    } // stages

}