# We use the "Secure Development Tools" which needs to be installed in any DevOps organization that use
# this YAML file. The free public Azure Pipelines for OSS includes these tools by default already.
# TODO: add ESLint once it's added to the "Secure Development Tools". TSLint is depricated.
variables:
- name: ubuntu_image
value: ubuntu-latest
- name: mac_image
value: macOS-latest
- name: windows_image
value: windows-latest
jobs:
- job: 'bld_native'
strategy:
matrix:
Linux:
image.name: '${{ variables.ubuntu_image }}'
Mac:
image.name: '${{ variables.mac_image }}'
Windows:
image.name: '${{ variables.windows_image }}'
maxParallel: 3
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: PythonScript@0
condition: startsWith(variables['image.name'], 'windows')
inputs:
scriptSource: 'inline'
script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe')
displayName: 'CUDA download installer'
- script: |
REM CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\
SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
REM the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
IF ERRORLEVEL 1 (
ECHO cuda_11.2.2_win10_network.exe FAILED
EXIT /B 201
)
.\build.bat
condition: startsWith(variables['image.name'], 'windows')
displayName: 'win bld_native'
- script: |
# sudo apt -y install nvidia-cuda-toolkit
# if [ $? -ne 0 ]; then
# exit 201
# fi
# nvcc --version
/bin/sh ./build.sh -asm
condition: startsWith(variables['image.name'], 'ubuntu')
displayName: 'linux bld_native'
- script: |
/bin/sh ./build.sh -asm
condition: startsWith(variables['image.name'], 'macOS')
displayName: 'mac bld_native'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: staging
artifactName: ebm_native-$(image.name)
displayName: 'Publish native shared library'
- task: PublishBuildArtifacts@1
condition: or(startsWith(variables['image.name'], 'ubuntu'), startsWith(variables['image.name'], 'macOS'))
continueOnError: true
inputs:
pathtoPublish: tmp/staging_asm_release_64
artifactName: asm-x64-$(image.name)
displayName: 'Publish assembly x64'
- task: PublishBuildArtifacts@1
condition: startsWith(variables['image.name'], 'macOS')
continueOnError: true
inputs:
pathtoPublish: tmp/staging_asm_release_arm
artifactName: asm-arm-$(image.name)
displayName: 'Publish assembly arm'
- job: 'bld_vis'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- script: npm install && npm run build-prod
workingDirectory: 'python/interpret-core/js'
displayName: 'bld_vis'
- task: CopyFiles@2
inputs:
sourceFolder: 'python/interpret-core/js/dist'
contents: 'interpret-inline.js'
targetFolder: '$(Build.ArtifactStagingDirectory)/vis'
displayName: 'Copy interpret-inline.js library'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)/vis
artifactName: interpret-inline-bundle
displayName: 'Publish interpret-inline.js library'
- job: 'bld_npm_package'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- script: npm install && npm run build-prod && npm pack
workingDirectory: 'python/interpret-core/js'
displayName: 'bld_npm_package'
- task: CopyFiles@2
inputs:
sourceFolder: 'python/interpret-core/js'
contents: '*.tgz'
targetFolder: '$(Build.ArtifactStagingDirectory)/npm'
displayName: 'Copy npm package'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/npm'
artifactName: 'npm'
displayName: 'Publish npm package'
- job: 'bld_R_package'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- script: |
sudo apt install --yes texlive-latex-base texlive-fonts-extra
if [ $? -ne 0 ]; then
exit 201
fi
Rscript build.R
workingDirectory: 'R'
displayName: 'Rscript build.R'
- script: cat tmp/R/interpret.Rcheck/00install.out
condition: failed()
displayName: 'Display errors'
- task: CopyFiles@2
inputs:
sourceFolder: staging
contents: 'interpret_*.tar.gz'
targetFolder: '$(Build.ArtifactStagingDirectory)/R'
displayName: 'Copy R package'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/R'
artifactName: 'R'
displayName: 'Publish R package'
- job: 'bld_sdist_package'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
architecture: 'x64'
- script: python -m pip install --upgrade pip setuptools wheel
displayName: 'Install tools'
- script: python setup.py sdist -d ../../staging
workingDirectory: 'python/interpret-core'
displayName: 'interpret-core bld_sdist_package'
- script: python setup.py sdist -d ../../staging
workingDirectory: 'python/interpret'
displayName: 'interpret bld_sdist_package'
- task: CopyFiles@2
inputs:
sourceFolder: staging
contents: '*.tar.gz'
targetFolder: '$(Build.ArtifactStagingDirectory)/sdist'
displayName: 'Copy sdist python package'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/sdist'
artifactName: 'sdist'
displayName: 'Publish sdist python package'
- job: 'docs'
dependsOn: 'bld_sdist_package'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadPipelineArtifact@2
inputs:
artifactName: 'sdist'
itemPattern: 'sdist/interpret-core-*.tar.gz'
targetPath: '$(System.ArtifactsDirectory)/sdist'
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
- script: |
cd $(System.ArtifactsDirectory)/sdist
tarball_path=$(echo interpret-core-*.tar.gz)
python -m pip install --upgrade pip setuptools wheel
python -m pip install "$tarball_path"[required,debug,notebook,plotly,lime,sensitivity,shap,ebm,linear,decisiontree,treeinterpreter,dash,skoperules,testing]
cd $(Build.SourcesDirectory)/docs/
pip install -r requirements.txt
/bin/sh ./build.sh
displayName: 'Build docs'
- task: PublishPipelineArtifact@1
inputs:
targetPath: 'docs/interpret_docs/_build/html/'
artifactName: 'docs'
displayName: 'Publish docs'
- job: 'bld_bdist_package'
dependsOn: ['bld_native', 'bld_vis']
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
architecture: 'x64'
- script: python -m pip install --upgrade pip setuptools wheel
displayName: 'Install tools'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*_x64*'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy x64 native library to python lib directory'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*_arm*'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy arm native library to python lib directory'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/interpret-inline.js'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy interpret-inline.js to python lib directory'
- script: python setup.py bdist_wheel -d ../../staging
workingDirectory: 'python/interpret-core'
displayName: 'interpret-core bld_bdist_package'
# TODO: python setup.py bdist_wheel -d ../../staging is rebuilding the native code and the javascript, which is slowing the build
- script: python setup.py bdist_wheel -d ../../staging
workingDirectory: 'python/interpret'
displayName: 'interpret bld_bdist_package'
- task: CopyFiles@2
inputs:
sourceFolder: staging
contents: '*.whl'
targetFolder: '$(Build.ArtifactStagingDirectory)/bdist'
displayName: 'Copy bdist python package'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/bdist'
artifactName: 'wheel'
displayName: 'Publish bdist python package'
- job: 'test_native'
dependsOn: 'bld_native'
strategy:
matrix:
Linux:
image.name: '${{ variables.ubuntu_image }}'
Mac:
image.name: '${{ variables.mac_image }}'
Windows:
image.name: '${{ variables.windows_image }}'
maxParallel: 3
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*'
targetFolder: 'staging'
flattenFolders: true
displayName: 'Copy native library to staging directory'
- task: PythonScript@0
condition: startsWith(variables['image.name'], 'windows')
inputs:
scriptSource: 'inline'
script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe')
displayName: 'Download CUDA installer'
- script: |
/bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -existing_debug_64 -existing_release_64
condition: startsWith(variables['image.name'], 'macOS')
displayName: 'mac test_native'
- script: |
/bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -debug_32 -release_32 -existing_debug_64 -existing_release_64 -no_valgrind
condition: and(startsWith(variables['image.name'], 'ubuntu'), ne(variables['Build.Reason'], 'Schedule'))
displayName: 'linux test_native (CI)'
- script: |
/bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -debug_32 -release_32 -existing_debug_64 -existing_release_64
condition: and(startsWith(variables['image.name'], 'ubuntu'), eq(variables['Build.Reason'], 'Schedule'))
displayName: 'linux test_native (Schedule)'
- script: |
REM CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\
SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
REM the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
IF ERRORLEVEL 1 (
ECHO cuda_11.2.2_win10_network.exe FAILED
EXIT /B 201
)
.\shared\ebm_native\ebm_native_test\ebm_native_test.bat -pipeline
condition: and(startsWith(variables['image.name'], 'windows'), ne(variables['Build.Reason'], 'Schedule'))
displayName: 'win test_native (CI)'
- script: |
REM CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\
SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
REM the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
IF ERRORLEVEL 1 (
ECHO cuda_11.2.2_win10_network.exe FAILED
EXIT /B 201
)
.\shared\ebm_native\ebm_native_test\ebm_native_test.bat -pipeline -analysis
condition: and(startsWith(variables['image.name'], 'windows'), eq(variables['Build.Reason'], 'Schedule'))
displayName: 'win test_native (Schedule)'
- task: ComponentGovernanceComponentDetection@0
displayName: 'Scan dependencies for incompatible licenses and security issues'
- task: CredScan@2
inputs:
toolMajorVersion: 'V2'
displayName: 'Scan codebase for leaked secrets'
condition: startsWith(variables['image.name'], 'windows')
# - task: BinSkim@3
# displayName: 'Scan DLLs for compiler/linker security improvements'
# condition: startsWith(variables['image.name'], 'windows')
# inputs:
# InputType: Basic
# AnalyzeTarget: 'staging\*.dll'
# AnalyzeSymPath: 'staging'
# AnalyzeVerbose: true
# - task: AntiMalware@3
# displayName: 'AntiMalware scan'
- task: SdtReport@1
condition: startsWith(variables['image.name'], 'windows')
inputs:
TsvFile: false
AllTools: false
BinSkim: false
CredScan: true
MSRD: false
RoslynAnalyzers: false
TSLint: false
ToolLogsNotFoundAction: 'Standard'
displayName: 'Generate security report'
- task: PublishSecurityAnalysisLogs@2
condition: startsWith(variables['image.name'], 'windows')
inputs:
ArtifactName: 'CodeAnalysisLogs'
ArtifactType: 'Container'
AllTools: true
ToolLogsNotFoundAction: 'Standard'
displayName: 'Publish security report'
- task: PostAnalysis@1
condition: startsWith(variables['image.name'], 'windows')
inputs:
AllTools: false
BinSkim: false
CredScan: true
RoslynAnalyzers: false
TSLint: false
ToolLogsNotFoundAction: 'Standard'
displayName: 'Stop on security errors'
- job: 'test' # just call this test since we have limited room in the UI to show the OS and python version
dependsOn: 'bld_bdist_package'
strategy:
matrix:
Linux38Python:
python.version: '3.8'
image.name: '${{ variables.ubuntu_image }}'
Linux39Python:
python.version: '3.9'
image.name: '${{ variables.ubuntu_image }}'
Linux310Python:
python.version: '3.10'
image.name: '${{ variables.ubuntu_image }}'
Win38Python:
python.version: '3.8'
image.name: '${{ variables.windows_image }}'
Win39Python:
python.version: '3.9'
image.name: '${{ variables.windows_image }}'
Win310Python:
python.version: '3.10'
image.name: '${{ variables.windows_image }}'
Mac38Python:
python.version: '3.8'
image.name: '${{ variables.mac_image }}'
Mac39Python:
python.version: '3.9'
image.name: '${{ variables.mac_image }}'
Mac310Python:
python.version: '3.10'
image.name: '${{ variables.mac_image }}'
maxParallel: 9
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy native library to python lib directory'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/interpret-inline.js'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy interpret-inline.js to python lib directory'
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
architecture: 'x64'
- script: python -m pip install --upgrade pip setuptools wheel
displayName: 'Install tools'
- script: python -m pip install -e .[required,debug,notebook,plotly,lime,sensitivity,shap,ebm,linear,decisiontree,treeinterpreter,dash,skoperules,testing]
workingDirectory: python/interpret-core
displayName: 'Install interpret-core package with requirements'
- script: |
set PATH=%PATH%;%GeckoWebDriver%
python -m pytest -vv -n auto --runslow --runselenium --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
workingDirectory: python/interpret-core
condition: and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master')))
displayName: 'pytest (win AND (scheduled OR master))'
- script: |
python -m pytest -vv -n auto --runslow --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
workingDirectory: python/interpret-core
condition: not(and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master'))))
displayName: 'pytest (rest)'
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testResultsFiles: '**/test-*.xml'
testRunTitle: 'Publish test results for Python $(python.version) at $(image.name)'
displayName: 'Publish test results'
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
# reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
condition: startsWith(variables['image.name'], 'windows')
displayName: 'Publish test coverage results'
- job: 'publish_source_code'
dependsOn: ['test_native', 'test']
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
architecture: 'x64'
- script: |
python -m pip install --upgrade black
black --check .
workingDirectory: 'python/interpret-core'
displayName: 'black formatting check'
- task: PublishPipelineArtifact@0
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
artifactName: 'source_code'
schedules:
- cron: "0 12 * * *"
displayName: Daily midnight build
branches:
include:
- develop
always: true