Commit 0faea9b3 authored by Mo Zhou's avatar Mo Zhou

New upstream version 0.18.1

parents

Too many changes to show.

To preserve performance only 1000 of 1000+ files are displayed.

// Copyright 2017-2019 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This script acts as a trigger script for the main ngraph-unittest.groovy
// Jenkins job. This script is part of a Jenkins multi-branch pipeline job
// which can trigger GitHub jobs more effectively than the GitHub Pull
// Request Builder (GHPRB) plugin, in our environment.
// The original ngraph-unittest job required the following parameters. We
// set these up below as global variables, so we do not need to rewrite the
// original script -- we only need to provide this new trigger hook.
//
// Parameters which ngraph-unittest uses:
String PR_URL = CHANGE_URL
String PR_COMMIT_AUTHOR = CHANGE_AUTHOR
String JENKINS_BRANCH = "master"
Integer TIMEOUTTIME = "3600"
// BRANCH parameter is no loner needed
// TRIGGER_URL parameter is no longer needed
// Constants
JENKINS_DIR = '.'
env.MB_PIPELINE_CHECKOUT = true
timestamps {
node("trigger") {
deleteDir() // Clear the workspace before starting
// Clone the cje-algo directory which contains our Jenkins groovy scripts
git(branch: JENKINS_BRANCH, changelog: false, poll: false,
url: 'https://github.intel.com/AIPG/cje-algo')
// Call the main job script.
//
// NOTE: We keep the main job script in github.intel.com because it may
// contain references to technology which has not yet been released.
//
echo "Calling ngraph-ci-premerge.groovy"
def ngraphCIPreMerge = load("${JENKINS_DIR}/ngraph-ci-premerge.groovy")
ngraphCIPreMerge(PR_URL, PR_COMMIT_AUTHOR, JENKINS_BRANCH, TIMEOUTTIME)
echo "ngraph-ci-premerge.groovy completed"
} // End: node
} // End: timestamps
echo "Done"
// INTEL CONFIDENTIAL
// Copyright 2018-2019 Intel Corporation All Rights Reserved.
// The source code contained or described herein and all documents related to the
// source code ("Material") are owned by Intel Corporation or its suppliers or
// licensors. Title to the Material remains with Intel Corporation or its
// suppliers and licensors. The Material may contain trade secrets and proprietary
// and confidential information of Intel Corporation and its suppliers and
// licensors, and is protected by worldwide copyright and trade secret laws and
// treaty provisions. No part of the Material may be used, copied, reproduced,
// modified, published, uploaded, posted, transmitted, distributed, or disclosed
// in any way without Intel's prior express written permission.
// No license under any patent, copyright, trade secret or other intellectual
// property right is granted to or conferred upon you by disclosure or delivery of
// the Materials, either expressly, by implication, inducement, estoppel or
// otherwise. Any license under such intellectual property rights must be express
// and approved by Intel in writing.
// Set LABEL variable if empty or not declared
try{ if(LABEL.trim() == "") {throw new Exception();} }catch(Exception e){LABEL="onnx && ci"}; echo "${LABEL}"
// CI settings and constants
PROJECT_NAME = "ngraph-onnx"
CI_ROOT = "ngraph/.ci/onnx/jenkins"
DOCKER_CONTAINER_NAME = "jenkins_${PROJECT_NAME}_ci"
NGRAPH_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph.git"
ONNX_GIT_ADDRESS = "https://github.com/NervanaSystems/ngraph-onnx.git"
JENKINS_GITHUB_CREDENTIAL_ID = "7157091e-bc04-42f0-99fd-dc4da2922a55"
ONNX_BRANCH = "$CHANGE_BRANCH"
GIT_PR_AUTHOR_EMAIL=""
GIT_COMMIT_AUTHOR_EMAIL=""
GIT_COMMIT_HASH=""
GIT_COMMIT_SUBJECT=""
// workaround for aborting previous builds on PR update
// TODO: Move to plugin based solution as soon as it's available
@NonCPS
def killPreviousRunningJobs() {
def jobname = env.JOB_NAME
def buildnum = env.BUILD_NUMBER.toInteger()
def job = Jenkins.instance.getItemByFullName(jobname)
for (build in job.builds) {
if (!build.isBuilding()){
continue;
}
if (buildnum == build.getNumber().toInteger()){
continue;
}
echo "Kill task = ${build}"
build.doStop();
}
}
def cloneRepository(String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
stage('Clone Repos') {
try {
sh "git clone $onnx_git_address -b $CHANGE_BRANCH ."
}
catch (Exception e) {
ONNX_BRANCH = "master"
sh """
echo "WARNING! Failed to clone ngraph-onnx branch $CHANGE_BRANCH ! Falling back to master."
echo "EXCEPTION: $e"
"""
checkout([$class: 'GitSCM',
branches: [[name: "master"]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
url: "${onnx_git_address}"]]])
}
dir ("ngraph") {
checkout([$class: 'GitSCM',
branches: [[name: "$CHANGE_BRANCH"]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${jenkins_github_credential_id}",
url: "${ngraph_git_address}"]]])
GIT_PR_AUTHOR_EMAIL = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim()
GIT_COMMIT_AUTHOR_EMAIL = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim()
GIT_COMMIT_HASH = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim()
GIT_COMMIT_SUBJECT = sh (script: 'git log -1 --pretty="format:%s" ', returnStdout: true).trim()
}
}
}
def buildImage(configurationMaps) {
Closure buildMethod = { configMap ->
sh """
${CI_ROOT}/utils/docker.sh build \
--name=${configMap["projectName"]} \
--version=${configMap["name"]} \
--dockerfile_path=${configMap["dockerfilePath"]}
"""
}
UTILS.createStage("Build_image", buildMethod, configurationMaps)
}
def runDockerContainers(configurationMaps) {
Closure runContainerMethod = { configMap ->
UTILS.propagateStatus("Build_image", configMap["name"])
sh """
mkdir -p ${HOME}/ONNX_CI
${CI_ROOT}/utils/docker.sh start \
--name=${configMap["projectName"]} \
--version=${configMap["name"]} \
--container_name=${configMap["dockerContainerName"]} \
--volumes="-v ${WORKSPACE}/${BUILD_NUMBER}:/logs -v ${HOME}/ONNX_CI:/home -v ${WORKDIR}:/root"
"""
}
UTILS.createStage("Run_docker_containers", runContainerMethod, configurationMaps)
}
def prepareEnvironment(configurationMaps) {
Closure prepareEnvironmentMethod = { configMap ->
UTILS.propagateStatus("Run_docker_containers", configMap["dockerContainerName"])
sh """
docker cp ${CI_ROOT}/utils/docker.sh ${configMap["dockerContainerName"]}:/home
docker exec ${configMap["dockerContainerName"]} bash -c "/root/${CI_ROOT}/prepare_environment.sh"
"""
}
UTILS.createStage("Prepare_environment", prepareEnvironmentMethod, configurationMaps)
}
def runToxTests(configurationMaps) {
Closure runToxTestsMethod = { configMap ->
UTILS.propagateStatus("Prepare_environment", configMap["dockerContainerName"])
sh """
NGRAPH_WHL=\$(docker exec ${configMap["dockerContainerName"]} find /root/ngraph/python/dist/ -name 'ngraph*.whl')
docker exec -e TOX_INSTALL_NGRAPH_FROM=\${NGRAPH_WHL} ${configMap["dockerContainerName"]} tox -c /root/
"""
}
UTILS.createStage("Run_tox_tests", runToxTestsMethod, configurationMaps)
}
def cleanup(configurationMaps) {
Closure cleanupMethod = { configMap ->
sh """
cd ${HOME}/ONNX_CI
./docker.sh chmod --container_name=${configMap["dockerContainerName"]} --directory="/logs" --options="-R 777" || true
./docker.sh stop --container_name=${configMap["dockerContainerName"]} || true
./docker.sh remove --container_name=${configMap["dockerContainerName"]} || true
./docker.sh clean_up || true
rm ${HOME}/ONNX_CI/docker.sh
rm -rf ${WORKSPACE}/${BUILD_NUMBER}
"""
}
UTILS.createStage("Cleanup", cleanupMethod, configurationMaps)
}
def notifyByEmail() {
configurationMaps = []
configurationMaps.add([
"name": "Notify"
])
String notifyPeople = "$GIT_PR_AUTHOR_EMAIL, $GIT_COMMIT_AUTHOR_EMAIL"
Closure notifyMethod = { configMap ->
if(currentBuild.result == "FAILURE") {
emailext (
subject: "NGraph-Onnx CI: NGraph PR $CHANGE_ID $currentBuild.result!",
body: """
<table style="width:100%">
<tr><td>Status:</td> <td>${currentBuild.result}</td></tr>
<tr><td>Pull Request Title:</td> <td>$CHANGE_TITLE</td></tr>
<tr><td>Pull Request:</td> <td><a href=$CHANGE_URL>$CHANGE_ID</a> </td></tr>
<tr><td>Branch:</td> <td>$CHANGE_BRANCH</td></tr>
<tr><td>Commit Hash:</td> <td>$GIT_COMMIT_SUBJECT</td></tr>
<tr><td>Commit Subject:</td> <td>$GIT_COMMIT_HASH</td></tr>
<tr><td>Jenkins Build:</td> <td> <a href=$RUN_DISPLAY_URL> ${BUILD_NUMBER} </a> </td></tr>
<tr><td>nGraph-ONNX Branch:</td> <td>${ONNX_BRANCH}</td></tr>
</table>
""",
to: "${notifyPeople}"
)
}
}
UTILS.createStage("Notify", notifyMethod, configurationMaps)
}
def main(String label, String projectName, String projectRoot, String dockerContainerName, String jenkins_github_credential_id, String ngraph_git_address, String onnx_git_address) {
killPreviousRunningJobs()
node(label) {
timeout(activity: true, time: 15) {
WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
def configurationMaps;
try {
dir ("${WORKDIR}") {
cloneRepository(jenkins_github_credential_id, ngraph_git_address, onnx_git_address)
// Load CI API
UTILS = load "${CI_ROOT}/utils/utils.groovy"
result = 'SUCCESS'
// Create configuration maps
configurationMaps = UTILS.getDockerEnvList(projectName, dockerContainerName, projectRoot)
// Execute CI steps
buildImage(configurationMaps)
runDockerContainers(configurationMaps)
prepareEnvironment(configurationMaps)
runToxTests(configurationMaps)
}
}
catch(e) {
// Set result to ABORTED if exception contains exit code of a process interrupted by SIGTERM
if ("$e".contains("143")) {
currentBuild.result = "ABORTED"