lint 7.84 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
pipeline {
    agent { label 'pr-lint-slave' }
    options {
        timestamps()
        ansiColor('xterm')
        timeout(time: 3, unit: 'HOURS') 
    }
    environment {
        PYENV_ROOT = "/usr/local/pyenv"
        PATH = "$PYENV_ROOT/bin:$PATH"
        PY_COLORS = 1
    }
    stages {
        stage('github-pending') {
            steps {
                githubNotify credentialsId: 'test-jenkins-credentials',
                    description: 'Python lint on changes begins...',
                    status: 'PENDING',
                    context: "jenkins/pr/lint"
            }
        }
        stage('setup') {
            steps {
                sh '''
                # Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M
                # -M is on by default in git 2.9+
                git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" > file-list-status.log
                # the -l increase the search limit, lets use awk so we do not need to repeat the search above.
                gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log
                gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log
                (git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log
                echo 254 > pylint-salt-chg.exit # assume failure
                echo 254 > pylint-salt-full.exit # assume failure
                echo 254 > pylint-tests-chg.exit # assume failure
                echo 254 > pylint-tests-full.exit # assume failure
                eval "$(pyenv init -)"
                pyenv --version
                pyenv install --skip-existing 2.7.14
                pyenv local 2.7.14
                pyenv shell 2.7.14
                python --version
                pip install tox
                '''
                archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log'
            }
        }
        stage('linting chg') {
            parallel {
                stage('lint salt chg') {
                    when {
                        expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/ }
                    }
                    steps {
                        sh '''
                        eval "$(pyenv init - --no-rehash)"
                        # tee makes the exit/return code always 0
                        grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | (xargs -r '--delimiter=\\n' tox -e pylint-salt ; echo "$?" >  pylint-salt-chg.exit) | tee pylint-report-salt-chg.log
                        # remove color escape coding
                        sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt-chg.log
                        read rc_exit < pylint-salt-chg.exit
                        exit "$rc_exit"
                        '''
                    }
                }
                stage('lint test chg') {
                    when {
                        expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/ }
                    }
                    steps {
                        sh '''
                        eval "$(pyenv init - --no-rehash)"
                        # tee makes the exit/return code always 0
                        grep -Ei '^tests/.*\\.py$' file-list-changed.log | (xargs -r '--delimiter=\\n' tox -e pylint-tests ; echo "$?" > pylint-tests-chg.exit) | tee pylint-report-tests-chg.log
                        # remove color escape coding
                        sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests-chg.log
                        read rc_exit < pylint-tests-chg.exit
                        exit "$rc_exit"
                        '''
                    }
                }
            }
            post {
                always {
                    archiveArtifacts artifacts: 'pylint-report-*-chg.log', allowEmptyArchive: true
                    step([$class: 'WarningsPublisher',
                        parserConfigurations: [[
                            parserName: 'PyLint',
                            pattern: 'pylint-report-*-chg.log'
                        ]],
                        failedTotalAll: '0',
                        useDeltaValues: false,
                        canRunOnFailed: true,
                        usePreviousBuildAsReference: true
                    ])
                }
            }
        }
        stage('linting all') {
            // perform a full linit if this is a merge forward and the change only lint passed.
            when {
                expression { return env.CHANGE_BRANCH =~ /(?i)^merge[._-]/ }
            }
            parallel {
                stage('setup full') {
                    steps {
                        githubNotify credentialsId: 'test-jenkins-credentials',
                            description: 'Python lint on everything begins...',
                            status: 'PENDING',
                            context: "jenkins/pr/lint"
                    }
                }
                stage('lint salt full') {
                    steps {
                        sh '''
                        eval "$(pyenv init - --no-rehash)"
                        (tox -e pylint-salt ; echo "$?" > pylint-salt-full.exit) | tee pylint-report-salt-full.log
                        # remove color escape coding
                        sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt-full.log
                        read rc_exit <  pylint-salt-full.exit
                        exit "$rc_exit"
                        '''
                    }
                }
                stage('lint test full') {
                    steps {
                        sh '''
                        eval "$(pyenv init - --no-rehash)"
                        (tox -e pylint-tests ; echo "$?" > pylint-tests-full.exit) | tee pylint-report-tests-full.log
                        # remove color escape coding
                        sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests-full.log
                        read rc_exit <  pylint-tests-full.exit
                        exit "$rc_exit"
                        '''
                    }
                }
            }
            post {
                always {
                    archiveArtifacts artifacts: 'pylint-report-*-full.log', allowEmptyArchive: true
                    step([$class: 'WarningsPublisher',
                        parserConfigurations: [[
                            parserName: 'PyLint',
                            pattern: 'pylint-report-*-full.log'
                        ]],
                        failedTotalAll: '0',
                        useDeltaValues: false,
                        canRunOnFailed: true,
                        usePreviousBuildAsReference: true
                    ])
                }
            }
        }
    }
    post {
        always {
            cleanWs()
        }
        success {
            githubNotify credentialsId: 'test-jenkins-credentials',
                description: 'Python lint test has passed',
                status: 'SUCCESS',
                context: "jenkins/pr/lint"
        }
        failure {
            githubNotify credentialsId: 'test-jenkins-credentials',
                description: 'Python lint test has failed',
                status: 'FAILURE',
                context: "jenkins/pr/lint"
            slackSend channel: "#jenkins-prod-pr", 
                color: '#FF0000', 
                message: "FAILED: PR-Job: '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})"
        }
    }
}