https://github.com/interpretml/interpret
Tip revision: b2365e7d1dd9ea81cada3285ccf9941f71a2a129 authored by Paul Koch on 14 March 2023, 06:06:03 UTC
added back the dash components to the installation since some versions of dash still require them, and updated the changelog for the v0.3.1 release
added back the dash components to the installation since some versions of dash still require them, and updated the changelog for the v0.3.1 release
Tip revision: b2365e7
azure-pipelines.yml
# We use the "Secure Development Tools" which needs to be installed in any DevOps organization that use
# this YAML file. The free public Azure Pipelines for OSS includes these tools by default already.
# TODO: add ESLint once it's added to the "Secure Development Tools". TSLint is depricated.
variables:
- name: ubuntu_image
value: ubuntu-latest
- name: mac_image
value: macOS-latest
- name: windows_image
value: windows-latest
jobs:
- job: 'bld_native'
strategy:
matrix:
Linux:
image.name: '${{ variables.ubuntu_image }}'
Mac:
image.name: '${{ variables.mac_image }}'
Windows:
image.name: '${{ variables.windows_image }}'
maxParallel: 3
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: PythonScript@0
condition: startsWith(variables['image.name'], 'windows')
inputs:
scriptSource: 'inline'
script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe')
displayName: 'CUDA download installer'
- script: |
REM CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\
SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
REM the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
IF ERRORLEVEL 1 (
ECHO cuda_11.2.2_win10_network.exe FAILED
EXIT /B 201
)
.\build.bat
condition: startsWith(variables['image.name'], 'windows')
displayName: 'win bld_native'
- script: |
# sudo apt -y install nvidia-cuda-toolkit
# if [ $? -ne 0 ]; then
# exit 201
# fi
# nvcc --version
/bin/sh ./build.sh -asm
condition: startsWith(variables['image.name'], 'ubuntu')
displayName: 'linux bld_native'
- script: |
/bin/sh ./build.sh -asm
condition: startsWith(variables['image.name'], 'macOS')
displayName: 'mac bld_native'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: staging
artifactName: ebm_native-$(image.name)
displayName: 'Publish native shared library'
- task: PublishBuildArtifacts@1
condition: or(startsWith(variables['image.name'], 'ubuntu'), startsWith(variables['image.name'], 'macOS'))
continueOnError: true
inputs:
pathtoPublish: tmp/staging_asm_release_64
artifactName: asm-x64-$(image.name)
displayName: 'Publish assembly x64'
- task: PublishBuildArtifacts@1
condition: startsWith(variables['image.name'], 'macOS')
continueOnError: true
inputs:
pathtoPublish: tmp/staging_asm_release_arm
artifactName: asm-arm-$(image.name)
displayName: 'Publish assembly arm'
- job: 'bld_vis'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- script: npm install && npm run build-prod
workingDirectory: 'python/interpret-core/js'
displayName: 'bld_vis'
- task: CopyFiles@2
inputs:
sourceFolder: 'python/interpret-core/js/dist'
contents: 'interpret-inline.js'
targetFolder: '$(Build.ArtifactStagingDirectory)/vis'
displayName: 'Copy interpret-inline.js library'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)/vis
artifactName: interpret-inline-bundle
displayName: 'Publish interpret-inline.js library'
- job: 'bld_npm_package'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- script: npm install && npm run build-prod && npm pack
workingDirectory: 'python/interpret-core/js'
displayName: 'bld_npm_package'
- task: CopyFiles@2
inputs:
sourceFolder: 'python/interpret-core/js'
contents: '*.tgz'
targetFolder: '$(Build.ArtifactStagingDirectory)/npm'
displayName: 'Copy npm package'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/npm'
artifactName: 'npm'
displayName: 'Publish npm package'
- job: 'bld_R_package'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- script: |
sudo apt install --yes texlive-latex-base texlive-fonts-extra
if [ $? -ne 0 ]; then
exit 201
fi
Rscript build.R
workingDirectory: 'R'
displayName: 'Rscript build.R'
- script: cat tmp/R/interpret.Rcheck/00install.out
condition: failed()
displayName: 'Display errors'
- task: CopyFiles@2
inputs:
sourceFolder: staging
contents: 'interpret_*.tar.gz'
targetFolder: '$(Build.ArtifactStagingDirectory)/R'
displayName: 'Copy R package'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/R'
artifactName: 'R'
displayName: 'Publish R package'
- job: 'bld_sdist_package'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
architecture: 'x64'
- script: python -m pip install --upgrade pip setuptools wheel
displayName: 'Install tools'
- script: python setup.py sdist -d ../../staging
workingDirectory: 'python/interpret-core'
displayName: 'interpret-core bld_sdist_package'
- script: python setup.py sdist -d ../../staging
workingDirectory: 'python/interpret'
displayName: 'interpret bld_sdist_package'
- task: CopyFiles@2
inputs:
sourceFolder: staging
contents: '*.tar.gz'
targetFolder: '$(Build.ArtifactStagingDirectory)/sdist'
displayName: 'Copy sdist python package'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/sdist'
artifactName: 'sdist'
displayName: 'Publish sdist python package'
- job: 'docs'
dependsOn: 'bld_sdist_package'
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadPipelineArtifact@2
inputs:
artifactName: 'sdist'
itemPattern: 'sdist/interpret-core-*.tar.gz'
targetPath: '$(System.ArtifactsDirectory)/sdist'
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
- script: |
cd $(System.ArtifactsDirectory)/sdist
tarball_path=$(echo interpret-core-*.tar.gz)
python -m pip install --upgrade pip setuptools wheel
python -m pip install "$tarball_path"[required,debug,notebook,plotly,lime,sensitivity,shap,ebm,linear,decisiontree,treeinterpreter,dash,skoperules,testing]
cd $(Build.SourcesDirectory)/docs/
pip install -r requirements.txt
/bin/sh ./build.sh
displayName: 'Build docs'
- task: PublishPipelineArtifact@1
inputs:
targetPath: 'docs/interpret_docs/_build/html/'
artifactName: 'docs'
displayName: 'Publish docs'
- job: 'bld_bdist_package'
dependsOn: ['bld_native', 'bld_vis']
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
architecture: 'x64'
- script: python -m pip install --upgrade pip setuptools wheel
displayName: 'Install tools'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*_x64*'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy x64 native library to python lib directory'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*_arm*'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy arm native library to python lib directory'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/interpret-inline.js'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy interpret-inline.js to python lib directory'
- script: python setup.py bdist_wheel -d ../../staging
workingDirectory: 'python/interpret-core'
displayName: 'interpret-core bld_bdist_package'
# TODO: python setup.py bdist_wheel -d ../../staging is rebuilding the native code and the javascript, which is slowing the build
- script: python setup.py bdist_wheel -d ../../staging
workingDirectory: 'python/interpret'
displayName: 'interpret bld_bdist_package'
- task: CopyFiles@2
inputs:
sourceFolder: staging
contents: '*.whl'
targetFolder: '$(Build.ArtifactStagingDirectory)/bdist'
displayName: 'Copy bdist python package'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/bdist'
artifactName: 'wheel'
displayName: 'Publish bdist python package'
- job: 'test_native'
dependsOn: 'bld_native'
strategy:
matrix:
Linux:
image.name: '${{ variables.ubuntu_image }}'
Mac:
image.name: '${{ variables.mac_image }}'
Windows:
image.name: '${{ variables.windows_image }}'
maxParallel: 3
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*'
targetFolder: 'staging'
flattenFolders: true
displayName: 'Copy native library to staging directory'
- task: PythonScript@0
condition: startsWith(variables['image.name'], 'windows')
inputs:
scriptSource: 'inline'
script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe')
displayName: 'Download CUDA installer'
- script: |
/bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -existing_debug_64 -existing_release_64
condition: startsWith(variables['image.name'], 'macOS')
displayName: 'mac test_native'
- script: |
/bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -debug_32 -release_32 -existing_debug_64 -existing_release_64 -no_valgrind
condition: and(startsWith(variables['image.name'], 'ubuntu'), ne(variables['Build.Reason'], 'Schedule'))
displayName: 'linux test_native (CI)'
- script: |
/bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -debug_32 -release_32 -existing_debug_64 -existing_release_64
condition: and(startsWith(variables['image.name'], 'ubuntu'), eq(variables['Build.Reason'], 'Schedule'))
displayName: 'linux test_native (Schedule)'
- script: |
REM CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\
SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
REM the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
IF ERRORLEVEL 1 (
ECHO cuda_11.2.2_win10_network.exe FAILED
EXIT /B 201
)
.\shared\ebm_native\ebm_native_test\ebm_native_test.bat -pipeline
condition: and(startsWith(variables['image.name'], 'windows'), ne(variables['Build.Reason'], 'Schedule'))
displayName: 'win test_native (CI)'
- script: |
REM CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\
SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
REM the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
IF ERRORLEVEL 1 (
ECHO cuda_11.2.2_win10_network.exe FAILED
EXIT /B 201
)
.\shared\ebm_native\ebm_native_test\ebm_native_test.bat -pipeline -analysis
condition: and(startsWith(variables['image.name'], 'windows'), eq(variables['Build.Reason'], 'Schedule'))
displayName: 'win test_native (Schedule)'
- task: ComponentGovernanceComponentDetection@0
displayName: 'Scan dependencies for incompatible licenses and security issues'
- task: CredScan@2
inputs:
toolMajorVersion: 'V2'
displayName: 'Scan codebase for leaked secrets'
condition: startsWith(variables['image.name'], 'windows')
# - task: BinSkim@3
# displayName: 'Scan DLLs for compiler/linker security improvements'
# condition: startsWith(variables['image.name'], 'windows')
# inputs:
# InputType: Basic
# AnalyzeTarget: 'staging\*.dll'
# AnalyzeSymPath: 'staging'
# AnalyzeVerbose: true
# - task: AntiMalware@3
# displayName: 'AntiMalware scan'
- task: SdtReport@1
condition: startsWith(variables['image.name'], 'windows')
inputs:
TsvFile: false
AllTools: false
BinSkim: false
CredScan: true
MSRD: false
RoslynAnalyzers: false
TSLint: false
ToolLogsNotFoundAction: 'Standard'
displayName: 'Generate security report'
- task: PublishSecurityAnalysisLogs@2
condition: startsWith(variables['image.name'], 'windows')
inputs:
ArtifactName: 'CodeAnalysisLogs'
ArtifactType: 'Container'
AllTools: true
ToolLogsNotFoundAction: 'Standard'
displayName: 'Publish security report'
- task: PostAnalysis@1
condition: startsWith(variables['image.name'], 'windows')
inputs:
AllTools: false
BinSkim: false
CredScan: true
RoslynAnalyzers: false
TSLint: false
ToolLogsNotFoundAction: 'Standard'
displayName: 'Stop on security errors'
- job: 'test' # just call this test since we have limited room in the UI to show the OS and python version
dependsOn: 'bld_bdist_package'
strategy:
matrix:
Linux38Python:
python.version: '3.8'
image.name: '${{ variables.ubuntu_image }}'
Linux39Python:
python.version: '3.9'
image.name: '${{ variables.ubuntu_image }}'
Linux310Python:
python.version: '3.10'
image.name: '${{ variables.ubuntu_image }}'
Win38Python:
python.version: '3.8'
image.name: '${{ variables.windows_image }}'
Win39Python:
python.version: '3.9'
image.name: '${{ variables.windows_image }}'
Win310Python:
python.version: '3.10'
image.name: '${{ variables.windows_image }}'
Mac38Python:
python.version: '3.8'
image.name: '${{ variables.mac_image }}'
Mac39Python:
python.version: '3.9'
image.name: '${{ variables.mac_image }}'
Mac310Python:
python.version: '3.10'
image.name: '${{ variables.mac_image }}'
maxParallel: 9
pool:
vmImage: '$(image.name)'
variables:
runCodesignValidationInjection: false
steps:
- task: DownloadBuildArtifacts@0
inputs:
buildType: 'current'
downloadType: 'specific'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/lib_ebm_native_*'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy native library to python lib directory'
- task: CopyFiles@2
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
contents: '**/interpret-inline.js'
targetFolder: 'python/interpret-core/interpret/lib'
flattenFolders: true
displayName: 'Copy interpret-inline.js to python lib directory'
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
architecture: 'x64'
- script: python -m pip install --upgrade pip setuptools wheel
displayName: 'Install tools'
- script: python -m pip install -e .[required,debug,notebook,plotly,lime,sensitivity,shap,ebm,linear,decisiontree,treeinterpreter,dash,skoperules,testing]
workingDirectory: python/interpret-core
displayName: 'Install interpret-core package with requirements'
- script: |
set PATH=%PATH%;%GeckoWebDriver%
python -m pytest -vv -n auto --runslow --runselenium --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
workingDirectory: python/interpret-core
condition: and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master')))
displayName: 'pytest (win AND (scheduled OR master))'
- script: |
python -m pytest -vv -n auto --runslow --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
workingDirectory: python/interpret-core
condition: not(and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master'))))
displayName: 'pytest (rest)'
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testResultsFiles: '**/test-*.xml'
testRunTitle: 'Publish test results for Python $(python.version) at $(image.name)'
displayName: 'Publish test results'
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
# reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
condition: startsWith(variables['image.name'], 'windows')
displayName: 'Publish test coverage results'
- job: 'publish_source_code'
dependsOn: ['test_native', 'test']
pool:
vmImage: '${{ variables.ubuntu_image }}'
variables:
runCodesignValidationInjection: false
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.8'
architecture: 'x64'
- script: |
python -m pip install --upgrade black
black --check .
workingDirectory: 'python/interpret-core'
displayName: 'black formatting check'
- task: PublishPipelineArtifact@0
inputs:
sourceFolder: '$(System.ArtifactsDirectory)'
artifactName: 'source_code'
schedules:
- cron: "0 12 * * *"
displayName: Daily midnight build
branches:
include:
- develop
always: true