Skip to content

Commit

Permalink
add windows wheels scripts, fix CI scripts and docs (#680)
Browse files Browse the repository at this point in the history 
Signed-off-by: Clement Fuji Tsang <[email protected]>
  • Loading branch information
@Caenorst
Caenorst authored Feb 1, 2023
1 parent 8d4718d commit 4fce007
Show file tree
Hide file tree
Showing 17 changed files with 560 additions and 286 deletions.
193 changes: 144 additions & 49 deletions ci/gitlab_jenkins_templates/core_ci.jenkins
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,38 +5,30 @@ import groovy.transform.Field
// Configs for build from pytorch docker images
// (See: https://hub.docker.com/r/pytorch/pytorch/tags)
def ubuntu_from_pytorch_configs = [
[
'cudaVer': '10.2', 'cudnnVer': '7', 'torchVer': '1.9.0',
'archsToTest': 'Tesla_V100_PCIE_32GB'
],
[
'cudaVer': '11.1', 'cudnnVer': '8', 'torchVer': '1.9.1',
'archsToTest': 'TITAN_RTX;A100_PCIE_40GB'
],
[
'cudaVer': '11.3', 'cudnnVer': '8', 'torchVer': '1.10.0',
'archsToTest': 'A100_PCIE_40GB'
'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.3', 'cudnnVer': '8', 'torchVer': '1.11.0',
'archsToTest': 'A100_PCIE_40GB'
'archsToTest': 'A10'
],
[
'cudaVer': '11.3', 'cudnnVer': '8', 'torchVer': '1.12.0',
'archsToTest': 'A100_PCIE_40GB'
'archsToTest': 'A30'
],
[
'cudaVer': '11.3', 'cudnnVer': '8', 'torchVer': '1.12.1',
'archsToTest': 'TITAN_RTX;A100_PCIE_40GB'
],
'archsToTest': 'A100_PCIE_40GB'
]
]

// Configs for build from NGC pytorch docker images
// (See: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch/tags)
def ubuntu_from_nvcr_configs = [
[
'baseImageTag': '22.10-py3',
'archsToTest': 'A100_PCIE_40GB'
'archsToTest': 'GA100-E4720-DVT'
],
]

Expand All @@ -45,13 +37,13 @@ def ubuntu_from_nvcr_configs = [
// (See: https://hub.docker.com/r/nvidia/cuda/tags)
def ubuntu_from_cuda_configs = [
[
'cudaVer': '10.2', 'cudnnVer': '7',
'pythonVer': '3.7', 'torchVer': '1.9.1',
'cudaVer': '11.3.0', 'cudnnVer': '8',
'pythonVer': '3.7', 'torchVer': '1.10.2',
'archsToTest': 'Tesla_V100_PCIE_32GB;TITAN_RTX'
],
[
'cudaVer': '11.6.1', 'cudnnVer': '8',
'pythonVer': '3.9', 'torchVer': '1.12.1',
'pythonVer': '3.9', 'torchVer': '1.13.1',
'archsToTest': 'A100_PCIE_40GB'
],
]
Expand All @@ -60,38 +52,31 @@ def ubuntu_from_cuda_configs = [
// (Use docker image ubuntu:18.04 as a base)
def ubuntu_cpuonly_configs = [
[
'pythonVer': '3.7', 'torchVer': '1.8.1',
'pythonVer': '3.7', 'torchVer': '1.10.0',
],
[
'pythonVer': '3.9', 'torchVer': '1.12.1',
'pythonVer': '3.9', 'torchVer': '1.13.1',
]
]

// Configs for building python wheels
def ubuntu_for_wheels_configs = [
/*
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.9.0', 'archsToTest': 'TITAN_V'
'torchVer': '1.10.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.9.0', 'archsToTest': 'TITAN_RTX'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.9.1', 'archsToTest': 'TITAN_V'
'torchVer': '1.10.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.9.1', 'archsToTest': 'TITAN_RTX'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.10.0', 'archsToTest': 'TITAN_V'
'torchVer': '1.10.1', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.10.0', 'archsToTest': 'Tesla_T4'
'torchVer': '1.10.2', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.3.1', 'cudnnVer': '8',
Expand Down Expand Up @@ -126,33 +111,119 @@ def ubuntu_for_wheels_configs = [
'torchVer': '1.11.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.3.1', 'cudnnVer': '8',
'cudaVer': '11.5.1', 'cudnnVer': '8',
'torchVer': '1.11.0', 'archsToTest': 'A100_PCIE_40GB'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.12.0', 'archsToTest': 'A100_PCIE_40GB'
'torchVer': '1.12.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.3.1', 'cudnnVer': '8',
'torchVer': '1.12.0', 'archsToTest': 'A100_PCIE_40GB'
'torchVer': '1.12.0', 'archsToTest': 'A10'
],
[
'cudaVer': '11.6.1', 'cudnnVer': '8',
'torchVer': '1.12.0', 'archsToTest': 'A100_PCIE_40GB'
'torchVer': '1.12.0', 'archsToTest': 'A30'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.12.1', 'archsToTest': 'A100_PCIE_40GB'
'torchVer': '1.12.1', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.3.1', 'cudnnVer': '8',
'torchVer': '1.12.1', 'archsToTest': 'A100_PCIE_40GB'
'torchVer': '1.12.1', 'archsToTest': 'A30'
],
[
'cudaVer': '11.6.1', 'cudnnVer': '8',
'torchVer': '1.12.1', 'archsToTest': 'GA100-E4720-DVT'
],
[
'cudaVer': '11.6.1', 'cudnnVer': '8',
'torchVer': '1.12.1', 'archsToTest': 'A100_PCIE_40GB'
'torchVer': '1.13.0', 'archsToTest': 'GA100-E4720-DVT'
],
[
'cudaVer': '11.7.1', 'cudnnVer': '8',
'torchVer': '1.13.0', 'archsToTest': 'A100_PCIE_40GB'
],
[
'cudaVer': '11.6.1', 'cudnnVer': '8',
'torchVer': '1.13.1', 'archsToTest': 'A10'
],
[
'cudaVer': '11.7.1', 'cudnnVer': '8',
'torchVer': '1.13.1', 'archsToTest': 'A30'
],
*/
]

def windows_for_wheels_configs = [
/*
[
'cudaVer': '11.1', 'cudnnVer': '8',
'torchVer': '1.10.0', 'archsToTest': ''
],
[
'cudaVer': '11.3', 'cudnnVer': '8',
'torchVer': '1.10.0', 'archsToTest': ''
],
[
'cudaVer': '11.1', 'cudnnVer': '8',
'torchVer': '1.10.1', 'archsToTest': ''
],
[
'cudaVer': '11.3', 'cudnnVer': '8',
'torchVer': '1.10.1', 'archsToTest': ''
],
[
'cudaVer': '11.1', 'cudnnVer': '8',
'torchVer': '1.10.2', 'archsToTest': ''
],
[
'cudaVer': '11.3', 'cudnnVer': '8',
'torchVer': '1.10.2', 'archsToTest': ''
],
[
'cudaVer': '11.3', 'cudnnVer': '8',
'torchVer': '1.11.0', 'archsToTest': ''
],
[
'cudaVer': '11.5', 'cudnnVer': '8',
'torchVer': '1.11.0', 'archsToTest': ''
],
[
'cudaVer': '11.3', 'cudnnVer': '8',
'torchVer': '1.12.0', 'archsToTest': ''
],
[
'cudaVer': '11.6', 'cudnnVer': '8',
'torchVer': '1.12.0', 'archsToTest': ''
],
[
'cudaVer': '11.3', 'cudnnVer': '8',
'torchVer': '1.12.1', 'archsToTest': ''
],
[
'cudaVer': '11.6', 'cudnnVer': '8',
'torchVer': '1.12.1', 'archsToTest': ''
],
[
'cudaVer': '11.6', 'cudnnVer': '8',
'torchVer': '1.13.0', 'archsToTest': ''
],
[
'cudaVer': '11.7', 'cudnnVer': '8',
'torchVer': '1.13.0', 'archsToTest': ''
],
[
'cudaVer': '11.6', 'cudnnVer': '8',
'torchVer': '1.13.1', 'archsToTest': ''
],
[
'cudaVer': '11.7', 'cudnnVer': '8',
'torchVer': '1.13.1', 'archsToTest': ''
],
*/
]

// Configs for build from Windows server docker images
Expand All @@ -163,13 +234,13 @@ def windows_from_server_configs = [
// test machines currently are the only option, named 'gpu_tester'
// two machines exist, only the TITAN RTX will pass tests
[
'cudaVer': '11.1',
'pythonVer': '3.7', 'torchVer': '1.8.2',
'cudaVer': '11.3',
'pythonVer': '3.7', 'torchVer': '1.10.0',
'archsToTest': 'gpu_tester' //'Tesla_V100_PCIE_32GB'
],
[
'cudaVer': '11.3',
'pythonVer': '3.8', 'torchVer': '1.12.1',
'cudaVer': '11.6',
'pythonVer': '3.9', 'torchVer': '1.13.1',
'archsToTest': 'gpu_tester' //'Tesla_V100_PCIE_32GB'
]
]
Expand Down Expand Up @@ -219,6 +290,23 @@ node {
)
}
}
for (config in windows_for_wheels_configs) {
for (pythonVer in ['3.7', '3.8', '3.9']) {

def cudaVerLabel = config['cudaVer'].split('\\.').join('')
def torchVerLabel = config['torchVer'].split('\\.').join('')
def pythonVerLabel = pythonVer.split('\\.').join('')
def configName = "windows-wheels-cuda${cudaVerLabel}-py${pythonVerLabel}-torch${torchVerLabel}"
jobMap["${configName}"] = prepareWindowsJob(
configName,
config['cudaVer'],
pythonVer,
config['torchVer'],
config['archsToTest'],
true
)
}
}
} else {
// Check if the last commit message has a [with custom] tag
def hasNoCustomInMess = sh(script: "git log -1 | grep '.*\\[with custom\\].*'",
Expand Down Expand Up @@ -271,7 +359,7 @@ node {
}

for (config in ubuntu_cpuonly_configs) {
def torchVerLabel = config['torchVer'].split('\\.')[0..1].join('')
def torchVerLabel = config['torchVer'].split('\\.').join('')
def pythonVerLabel = config['pythonVer'].split('\\.').join('')
def configName = "cpuonly-py${config['pythonVer']}-torch${config['torchVer']}"
jobMap["${configName}"] = prepareUbuntuCPUOnlyJob(
Expand All @@ -283,15 +371,16 @@ node {

for (config in windows_from_server_configs) {
def cudaVerLabel = config['cudaVer'].split('\\.').join('')
def torchVerLabel = config['torchVer'].split('\\.')[0..1].join('')
def torchVerLabel = config['torchVer'].split('\\.').join('')
def pythonVerLabel = config['pythonVer'].split('\\.').join('')
def configName = "windows-cuda${cudaVerLabel}-py${pythonVerLabel}-torch${torchVerLabel}"
jobMap["${configName}"] = prepareWindowsJob(
configName,
config['cudaVer'],
config['pythonVer'],
config['torchVer'],
config['archsToTest']
config['archsToTest'],
false
)
}
}
Expand Down Expand Up @@ -389,12 +478,17 @@ def prepareUbuntuCPUOnlyJob(configName, pythonVer, torchVer) {
}
}

def prepareWindowsJob(configName, cudaVer, pythonVer, torchVer, archsToTest) {
def prepareWindowsJob(configName, cudaVer, pythonVer, torchVer, archsToTest,
buildWheel) {
return {
stage("${configName}") {
updateGitlabCommitStatus(name: "build-${configName}", state: "pending")
for (arch in archsToTest.split(';')) {
updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: "pending")
if (buildWheel.toBoolean()) {
updateGitlabCommitStatus(name: "test-${configName}", state: "pending")
} else {
for (arch in archsToTest.split(';')) {
updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: "pending")
}
}
build job: "windows_build_template_CI",
parameters: [
Expand All @@ -407,7 +501,8 @@ def prepareWindowsJob(configName, cudaVer, pythonVer, torchVer, archsToTest) {
string(name: 'sourceBranch', value: "${gitlabSourceBranch}"),
string(name: 'repoUrl', value: "${scm.userRemoteConfigs[0].url}" ),
string(name: 'commitHash', value: "${commitHash}"),
string(name: 'archsToTest', value: "${archsToTest}")
string(name: 'archsToTest', value: "${archsToTest}"),
booleanParam(name: 'buildWheel', value: "${buildWheel}")
],
wait: false,
propagate: false
Expand Down
16 changes: 6 additions & 10 deletions ci/gitlab_jenkins_templates/ubuntu_custom_build_CI.jenkins
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,34 +93,30 @@ spec:
cudaTag = cudaVer.split('\\.')[0..<2].join('')
targetImage.inside("-v /mount_binaries:/mount_binaries") {
sh """
mkdir -p /mount_binaries/torch-${torchVer}+cu${cudaTag}
python setup.py bdist_wheel --dist-dir /mount_binaries/torch-${torchVer}+cu${cudaTag}
ls .
python setup.py bdist_wheel --dist-dir .
"""
}
pythonVerTag = pythonVer.split('\\.').join('')
Integer pythonVerTagVal = pythonVerTag
sh """echo ${pythonVerTagVal}"""
pythonVerAbiTag = (pythonVerTagVal < 38) ? pythonVerTag + 'm' : pythonVerTag
kaolinVer = sh(script: "cat ./version.txt", returnStdout: true).trim()
baseWheelName = "kaolin-${kaolinVer}-cp${pythonVerTag}-cp${pythonVerAbiTag}"
srcWheelName = "${baseWheelName}-linux_x86_64.whl"
dstWheelName = "${baseWheelName}-manylinux1_x86_64.whl"
sh """
mv /mount_binaries/torch-${torchVer}+cu${cudaTag}/${srcWheelName} ./${dstWheelName}
"""
wheelName = "${baseWheelName}-linux_x86_64.whl"
}
stage("Reinstall from wheel") {
targetImage = docker.build(
"${targetImageTag}",
"""--no-cache --network host -f ./tools/linux/Dockerfile.install_wheel \
--build-arg BASE_IMAGE=${targetImageTag}-base \
--build-arg WHEEL_NAME=${dstWheelName} \
--build-arg WHEEL_NAME=${wheelName} \
.
""")
}
stage("Push wheel to volume") {
sh """
mv ./${dstWheelName} /mount_binaries/torch-${torchVer}+cu${cudaTag}/${dstWheelName}
mkdir -p /mount_binaries/torch-${torchVer}_cu${cudaTag}
mv ./${wheelName} /mount_binaries/whl/torch-${torchVer}_cu${cudaTag}/${wheelName}
"""
}
}
Expand Down
4 changes: 2 additions & 2 deletions ci/gitlab_jenkins_templates/ubuntu_test_CI.jenkins
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ spec:
""") {
node(POD_LABEL) {
container("docker") {
timeout(time: 25, unit: 'MINUTES') {
timeout(time: 30, unit: 'MINUTES') {
updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: "running")
stage("Install deps") {
sh 'pip install -r /kaolin/tools/ci_requirements.txt'
Expand All @@ -56,7 +56,7 @@ spec:
stage("Pytest") {
sh '''
export KAOLIN_TEST_NVIDIFFRAST=1
pytest -s --cov=/kaolin/kaolin /kaolin/tests/python/kaolin
pytest -s --cov=/kaolin/kaolin /kaolin/tests/python/kaolin/
'''
}
} catch(e) {
Expand Down
Loading

0 comments on commit 4fce007

Please sign in to comment.