# We use the "Secure Development Tools" which needs to be installed in any DevOps organization that use
# this YAML file. The free public Azure Pipelines for OSS includes these tools by default already.
# TODO: add ESLint once it's added to the "Secure Development Tools". TSLint is depricated.
variables:
- name: ubuntu_image
value: ubuntu-18.04
- name: mac_image
value: macOS-10.14
- name: windows_image
value: windows-2019
jobs:
- job: 'Build_ebm_native'
strategy:
matrix:
Linux:
image.name: '${{ variables.ubuntu_image }}'
Mac:
image.name: '${{ variables.mac_image }}'
Windows:
image.name: '${{ variables.windows_image }}'
maxParallel: 3
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: PythonScript@0
condition: startsWith(variables['image.name'], 'windows')
inputs:
scriptSource: 'inline'
script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe')
- script: |
# CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
set PATH=%PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\
set CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
# the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
.\build.bat
condition: startsWith(variables['image.name'], 'windows')
displayName: 'Building ebm_native code (Windows)'
- script: |
sudo apt -y install nvidia-cuda-toolkit
nvcc --version
/bin/sh ./build.sh
condition: startsWith(variables['image.name'], 'ubuntu')
displayName: 'Building ebm_native code (Linux)'
- script: |
/bin/sh ./build.sh
condition: startsWith(variables['image.name'], 'macOS')
displayName: 'Building ebm_native code (Mac)'
- task: CopyFiles@2
condition: succeeded()
inputs:
sourceFolder: staging
contents: '*'
targetFolder: '$(Build.ArtifactStagingDirectory)'
displayName: 'Move binary to staging'
- task: PublishBuildArtifacts@1
condition: succeeded()
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)
artifactName: ebm_native-$(image.name)
displayName: 'Publish ebm_native library'
- job: 'Test_ebm_native'
dependsOn: 'Build_ebm_native'
strategy:
matrix:
Linux:
image.name: '${{ variables.ubuntu_image }}'
Mac:
image.name: '${{ variables.mac_image }}'
Windows:
image.name: '${{ variables.windows_image }}'
maxParallel: 3
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
displayName: 'Download build artifacts'
- task: CopyFiles@2
condition: succeeded()
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*'
targetFolder: 'staging'
flattenFolders: true
displayName: 'Move binary to staging directory'
- task: PythonScript@0
condition: startsWith(variables['image.name'], 'windows')
inputs:
scriptSource: 'inline'
script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe')
- script: |
# CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
set PATH=%PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\
set CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
# the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
.\shared\ebm_native\ebm_native_test\ebm_native_test.bat -pipeline
condition: and(startsWith(variables['image.name'], 'windows'), eq(variables['Build.Reason'], 'Schedule'))
displayName: 'Testing ebm_native code (Windows)'
- script: |
/bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -pipeline
condition: and(startsWith(variables['image.name'], 'ubuntu'), eq(variables['Build.Reason'], 'Schedule'))
displayName: 'Testing ebm_native code (Linux)'
- script: |
/bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -pipeline
condition: startsWith(variables['image.name'], 'macOS')
displayName: 'Testing ebm_native code (Mac)'
- task: ComponentGovernanceComponentDetection@0
displayName: 'Scan dependencies for incompatible licenses and security issues'
- task: CredScan@2
displayName: 'Scan codebase for leaked secrets'
condition: startsWith(variables['image.name'], 'windows')
# - task: BinSkim@3
# displayName: 'Scan DLLs for compiler/linker security improvements'
# condition: startsWith(variables['image.name'], 'windows')
# inputs:
# InputType: Basic
# AnalyzeTarget: 'staging\*.dll'
# AnalyzeSymPath: 'staging'
# AnalyzeVerbose: true
# - task: AntiMalware@3
# displayName: 'AntiMalware scan'
- task: SdtReport@1
displayName: 'Generate security report'
condition: startsWith(variables['image.name'], 'windows')
inputs:
TsvFile: false
AllTools: false
BinSkim: false
CredScan: true
MSRD: false
RoslynAnalyzers: false
TSLint: false
ToolLogsNotFoundAction: 'Standard'
- task: PublishSecurityAnalysisLogs@2
displayName: 'Publish security report'
condition: startsWith(variables['image.name'], 'windows')
inputs:
ArtifactName: 'CodeAnalysisLogs'
ArtifactType: 'Container'
AllTools: true
ToolLogsNotFoundAction: 'Standard'
- task: PostAnalysis@1
displayName: 'Stop on security errors'
condition: startsWith(variables['image.name'], 'windows')
inputs:
AllTools: false
BinSkim: false
CredScan: true
RoslynAnalyzers: false
TSLint: false
ToolLogsNotFoundAction: 'Standard'
- job: 'Build_JS_Inline'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.6'
architecture: 'x64'
- script: python build-js-bundles.py devops
workingDirectory: 'python/scripts'
displayName: 'Build JS bundle'
- task: CopyFiles@2
condition: succeeded()
inputs:
sourceFolder: 'python/interpret-core/interpret/lib'
contents: 'interpret-inline.js'
targetFolder: '$(Build.ArtifactStagingDirectory)'
displayName: 'Move assets to staging'
- task: PublishBuildArtifacts@1
condition: succeeded()
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)
artifactName: interpret-inline-bundle
displayName: 'Publish interpret-inline.js library'
- job: 'Build_Package'
dependsOn: ['Build_ebm_native', 'Build_JS_Inline']
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
displayName: 'Download build artifacts'
- task: UsePythonVersion@0
condition: succeeded()
inputs:
versionSpec: '3.6'
architecture: 'x64'
- script: python -m pip install --upgrade pip setuptools wheel
condition: succeeded()
displayName: 'Install tools'
- task: CopyFiles@2
condition: succeeded()
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*_x64*'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Move binary to Python layer'
- task: CopyFiles@2
condition: succeeded()
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/interpret-inline.js'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Move interpret-inline.js to Python layer'
- script: python setup.py bdist_wheel -d ../../staging
condition: succeeded()
workingDirectory: 'python/interpret-api'
displayName: 'Build wheel (interpret-api)'
- script: python setup.py bdist_wheel -d ../../staging
condition: succeeded()
workingDirectory: 'python/interpret-core'
displayName: 'Build wheel (interpret-core)'
- script: python setup.py bdist_wheel -d ../../staging
condition: succeeded()
workingDirectory: 'python/interpret'
displayName: 'Build wheel (interpret)'
- task: CopyFiles@2
condition: succeeded()
inputs:
sourceFolder: staging
contents: '*.whl'
targetFolder: '$(Build.ArtifactStagingDirectory)/wheel'
displayName: 'Move wheel for Build Artifact'
- task: PublishBuildArtifacts@1
condition: succeeded()
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/wheel'
artifactName: 'wheel'
displayName: 'Publish wheel as Build Artifact'
- job: 'Test'
dependsOn: 'Build_Package'
# NOTE: Python 3.9 not supported yet in Azure DevOps.
strategy:
matrix:
LinuxPython36:
python.version: '3.6'
image.name: '${{ variables.ubuntu_image }}'
LinuxPython37:
python.version: '3.7'
image.name: '${{ variables.ubuntu_image }}'
LinuxPython38:
python.version: '3.8'
image.name: '${{ variables.ubuntu_image }}'
# LinuxPython39:
# python.version: '3.9'
# image.name: '${{ variables.ubuntu_image }}'
WindowsPython36:
python.version: '3.6'
image.name: '${{ variables.windows_image }}'
WindowsPython37:
python.version: '3.7'
image.name: '${{ variables.windows_image }}'
WindowsPython38:
python.version: '3.8'
image.name: '${{ variables.windows_image }}'
# WindowsPython39:
# python.version: '3.9'
# image.name: '${{ variables.windows_image }}'
MacPython36:
python.version: '3.6'
image.name: '${{ variables.mac_image }}'
MacPython37:
python.version: '3.7'
image.name: '${{ variables.mac_image }}'
MacPython38:
python.version: '3.8'
image.name: '${{ variables.mac_image }}'
# MacPython39:
# python.version: '3.9'
# image.name: '${{ variables.mac_image }}'
maxParallel: 9
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
displayName: 'Download build artifacts'
- task: CopyFiles@2
condition: succeeded()
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Move binary to Python layer'
- task: CopyFiles@2
condition: succeeded()
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/interpret-inline.js'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Move Interpret inline to Python layer'
- task: UsePythonVersion@0
condition: succeeded()
inputs:
versionSpec: '$(python.version)'
architecture: 'x64'
- script: python -m pip install --upgrade pip setuptools wheel
condition: succeeded()
displayName: 'Install tools'
- script: python -m pip install numpy scipy
condition: succeeded()
workingDirectory: python/interpret-core
displayName: 'Install numpy/scipy first for mis-specified pip packages.'
- script: |
mkdir -p ~/.matplotlib
echo "backend: TkAgg" >> ~/.matplotlib/matplotlibrc
condition: startsWith(variables['image.name'], 'macOS')
displayName: 'Matplotlib patch for mac.'
# - script: python -m pip install -r dev-requirements.txt
- script: python -m pip install -e .
condition: succeeded()
workingDirectory: python/interpret-api
displayName: 'Install API requirements (Linux/Mac/Windows)'
- script: python -m pip install -e .[required,debug,notebook,plotly,lime,sensitivity,shap,ebm,linear,decisiontree,treeinterpreter,dash,skoperules,testing]
condition: succeeded()
workingDirectory: python/interpret-core
displayName: 'Install requirements (Linux/Mac/Windows)'
- script: |
set PATH=%PATH%;%GeckoWebDriver%
python -m pytest -vv -n auto --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
workingDirectory: python/interpret-api
displayName: 'Run pytest on API'
- script: |
set PATH=%PATH%;%GeckoWebDriver%
python -m pytest -vv -n auto --runslow --runselenium --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
workingDirectory: python/interpret-core
condition: and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master')))
displayName: 'Run pytest (Windows scheduled / master)'
- script: |
python -m pytest -vv -n auto --runslow --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
workingDirectory: python/interpret-core
condition: not(and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master'))))
displayName: 'Run pytest (Rest)'
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testResultsFiles: '**/test-*.xml'
testRunTitle: 'Publish test results for Python $(python.version) at $(image.name)'
displayName: 'Publish test results'
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
condition: startsWith(variables['image.name'], 'windows')
displayName: 'Publish test coverage results'
- job: 'Publish_Package'
# dependsOn: ['Test_ebm_native', 'Test']
dependsOn: ['Test']
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'single'
artifactName: 'wheel'
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'single'
artifactName: 'interpret-inline-bundle'
- task: PublishPipelineArtifact@0
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
artifactName: 'drop'
schedules:
- cron: "0 12 * * *"
displayName: Daily midnight build
branches:
include:
- develop
always: true