https://github.com/interpretml/interpret
Tip revision: b2365e7d1dd9ea81cada3285ccf9941f71a2a129 authored by Paul Koch on 14 March 2023, 06:06:03 UTC
added back the dash components to the installation since some versions of dash still require them, and updated the changelog for the v0.3.1 release
added back the dash components to the installation since some versions of dash still require them, and updated the changelog for the v0.3.1 release
Tip revision: b2365e7
build.sh
#!/bin/sh
# TODO also build our html resources here, and also in the .bat file for Windows
sanitize() {
# use this techinque where single quotes are expanded to '\'' (end quotes insert single quote, start quote)
# but fixed from the version in this thread:
# https://stackoverflow.com/questions/15783701/which-characters-need-to-be-escaped-when-using-bash
# https://stackoverflow.com/questions/17529220/why-should-eval-be-avoided-in-bash-and-what-should-i-use-instead
printf "%s" "$1" | sed "s/'/'\\\\''/g; 1s/^/'/; \$s/\$/'/"
}
get_file_body() {
# https://www.oncrashreboot.com/use-sed-to-split-path-into-filename-extension-and-directory
printf "%s" "$1" | sed 's/\(.*\)\/\(.*\)\.\(.*\)$/\2/'
}
check_install() {
l1_tmp_path_unsanitized="$1"
l1_package="$2"
if [ ! -f "$l1_tmp_path_unsanitized/$l1_package.chk" ]; then
printf "%s\n" "Installing $l1_package"
sudo apt-get -y install "$l1_package"
l1_ret_code=$?
if [ $l1_ret_code -ne 0 ]; then
exit $l1_ret_code
fi
# write out an empty file to signal that this has been installed
printf "" > "$l1_tmp_path_unsanitized/$l1_package.chk"
l1_ret_code=$?
if [ $l1_ret_code -ne 0 ]; then
exit $l1_ret_code
fi
fi
}
make_initial_paths_simple() {
l2_obj_path_unsanitized="$1"
l2_bin_path_unsanitized="$2"
[ -d "$l2_obj_path_unsanitized" ] || mkdir -p "$l2_obj_path_unsanitized"
l2_ret_code=$?
if [ $l2_ret_code -ne 0 ]; then
exit $l2_ret_code
fi
[ -d "$l2_bin_path_unsanitized" ] || mkdir -p "$l2_bin_path_unsanitized"
l2_ret_code=$?
if [ $l2_ret_code -ne 0 ]; then
exit $l2_ret_code
fi
}
compile_file() {
l3_compiler="$1"
l3_compiler_args_sanitized="$2"
l3_file_unsanitized="$3"
l3_obj_path_unsanitized="$4"
l3_asm="$5"
l3_zone="$6"
l3_file_sanitized=`sanitize "$l3_file_unsanitized"`
l3_file_body_unsanitized=`get_file_body "$l3_file_unsanitized"`
l3_object_full_file_unsanitized="$l3_obj_path_unsanitized/${l3_file_body_unsanitized}_$l3_zone.o"
l3_object_full_file_sanitized=`sanitize "$l3_object_full_file_unsanitized"`
g_all_object_files_sanitized="$g_all_object_files_sanitized $l3_object_full_file_sanitized"
l3_compile_specific="$l3_compiler $l3_compiler_args_sanitized -c $l3_file_sanitized -o $l3_object_full_file_sanitized 2>&1"
l3_compile_out=`eval "$l3_compile_specific"`
l3_ret_code=$?
g_compile_out_full="$g_compile_out_full$l3_compile_out"
if [ $l3_ret_code -ne 0 ]; then
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
exit $l3_ret_code
fi
if [ $l3_asm -ne 0 ]; then
# - I'd rather do our real compile above with no special parameters because I'm not confident the compiler would
# produce the same output if we included extra debugger info disassembly commands. It's better to stick with
# the normal program flow for our shared library output. This rules out: --save-temps=obj
# -Wa,-adhln=myoutput.s can be used in-stream, but we've ruled this out per above. We can also use it
# to generate the .s file below, but I found that this didn't have much benefit over -S and -fverbose-asm
# We also write out objdump disassembly from the final library output itself which should allow us to
# check that this annotated assembly is the same as what gets finally generated
# - https://panthema.net/2013/0124-GCC-Output-Assembler-Code/
# - https://stackoverflow.com/questions/137038/how-do-you-get-assembler-output-from-c-c-source-in-gcc
# - https://linux.die.net/man/1/as
# If this fails then ignore the error and we'll just be missing this file.
l3_asm_full_file_unsanitized="$l3_obj_path_unsanitized/${l3_file_body_unsanitized}_$l3_zone.s"
l3_asm_full_file_sanitized=`sanitize "$l3_asm_full_file_unsanitized"`
l3_compile_specific_asm="$l3_compiler $l3_compiler_args_sanitized -fverbose-asm -S $l3_file_sanitized -o $l3_asm_full_file_sanitized 2>&1"
l3_compile_out_asm=`eval "$l3_compile_specific_asm"`
fi
}
compile_directory_c() {
l4_compiler="$1"
l4_compiler_args_sanitized="$2"
l4_src_path_unsanitized="$3"
l4_obj_path_unsanitized="$4"
l4_asm="$5"
l4_zone="$6"
# zsh (default shell in macs) terminates if you try to glob expand zero results, so check first
find "$l4_src_path_unsanitized" -maxdepth 1 -type f -name '*.c' 2>/dev/null | grep -q .
l4_ret_code=$?
if [ $l4_ret_code -eq 0 ]; then
# use globs with preceeding directory per: https://dwheeler.com/essays/filenames-in-shell.html
for l4_file_unsanitized in "$l4_src_path_unsanitized"/*.c ; do
# glob expansion returns *.c when there are no matches, so we need to check for the existance of the file
if [ -f "$l4_file_unsanitized" ] ; then
compile_file "$l4_compiler" "$l4_compiler_args_sanitized" "$l4_file_unsanitized" "$l4_obj_path_unsanitized" "$l4_asm" "$l4_zone"
fi
done
fi
}
compile_directory_cpp() {
l5_compiler="$1"
l5_compiler_args_sanitized="$2"
l5_src_path_unsanitized="$3"
l5_obj_path_unsanitized="$4"
l5_asm="$5"
l5_zone="$6"
# zsh (default shell in macs) terminates if you try to glob expand zero results, so check first
find "$l5_src_path_unsanitized" -maxdepth 1 -type f -name '*.cpp' 2>/dev/null | grep -q .
l5_ret_code=$?
if [ $l5_ret_code -eq 0 ]; then
# use globs with preceeding directory per: https://dwheeler.com/essays/filenames-in-shell.html
for l5_file_unsanitized in "$l5_src_path_unsanitized"/*.cpp ; do
# glob expansion returns *.cpp when there are no matches, so we need to check for the existance of the file
if [ -f "$l5_file_unsanitized" ] ; then
compile_file "$l5_compiler" "$l5_compiler_args_sanitized" "$l5_file_unsanitized" "$l5_obj_path_unsanitized" "$l5_asm" "$l5_zone"
fi
done
fi
}
compile_compute() {
l6_compiler="$1"
l6_compiler_args_sanitized="$2"
l6_src_path_sanitized="$3"
l6_src_path_unsanitized="$4"
l6_obj_path_unsanitized="$5"
l6_asm="$6"
l6_zone="$7"
compile_directory_cpp "$l6_compiler" "$l6_compiler_args_sanitized -DZONE_$l6_zone" "$l6_src_path_unsanitized/compute" "$l6_obj_path_unsanitized" "$l6_asm" "$l6_zone"
compile_directory_cpp "$l6_compiler" "$l6_compiler_args_sanitized -I$l6_src_path_sanitized/compute/${l6_zone}_ebm -DZONE_$l6_zone" "$l6_src_path_unsanitized/compute/${l6_zone}_ebm" "$l6_obj_path_unsanitized" "$l6_asm" "$l6_zone"
}
link_file() {
l7_linker="$1"
l7_linker_args_sanitized="$2"
l7_bin_path_unsanitized="$3"
l7_bin_file="$4"
l7_bin_path_sanitized=`sanitize "$l7_bin_path_unsanitized"`
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
l7_compile_specific="$l7_linker $g_all_object_files_sanitized $l7_linker_args_sanitized -o $l7_bin_path_sanitized/$l7_bin_file 2>&1"
l7_compile_out=`eval "$l7_compile_specific"`
l7_ret_code=$?
g_compile_out_full="$g_compile_out_full$l7_compile_out"
if [ $l7_ret_code -ne 0 ]; then
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
exit $l7_ret_code
fi
}
copy_bin_files() {
l8_bin_path_unsanitized="$1"
l8_bin_file="$2"
l8_python_lib_unsanitized="$3"
l8_staging_path_unsanitized="$4"
cp "$l8_bin_path_unsanitized/$l8_bin_file" "$l8_python_lib_unsanitized/"
l8_ret_code=$?
if [ $l8_ret_code -ne 0 ]; then
exit $l8_ret_code
fi
cp "$l8_bin_path_unsanitized/$l8_bin_file" "$l8_staging_path_unsanitized/"
l8_ret_code=$?
if [ $l8_ret_code -ne 0 ]; then
exit $l8_ret_code
fi
}
copy_asm_files() {
l9_obj_path_unsanitized="$1"
l9_tmp_path_unsanitized="$2"
l9_bin_file_unsanitized="$3"
l9_staging_tag="$4"
l9_asm="$5"
if [ $l9_asm -ne 0 ]; then
l9_tagged_path_unsanitized="$l9_tmp_path_unsanitized/staging_$l9_staging_tag"
[ -d "$l9_tagged_path_unsanitized" ] || mkdir -p "$l9_tagged_path_unsanitized"
l9_ret_code=$?
if [ $l9_ret_code -ne 0 ]; then
exit $l9_ret_code
fi
cp "$l9_obj_path_unsanitized"/*.s "$l9_tagged_path_unsanitized/"
l9_ret_code=$?
if [ $l9_ret_code -ne 0 ]; then
exit $l9_ret_code
fi
#also generate a disassembly from the final output that we can compare the individual files against
l9_bin_file_body_unsanitized=`get_file_body "$l9_bin_file_unsanitized"`
os_type=`uname`
if [ "$os_type" = "Linux" ]; then
# - https://stackoverflow.com/questions/1289881/using-gcc-to-produce-readable-assembly
# GNU objdump https://linux.die.net/man/1/objdump
objdump --disassemble --private-headers --reloc --dynamic-reloc --section-headers --syms --line-numbers --no-show-raw-insn --source "$l9_bin_file_unsanitized" > "$l9_tagged_path_unsanitized/$l9_bin_file_body_unsanitized.s"
elif [ "$os_type" = "Darwin" ]; then
# objdump on mac is actually llvm-objdump
# https://llvm.org/docs/CommandGuide/llvm-objdump.html
# otool might be a better choice on mac, but this does what we need in combination with the individual
# module assembly, so keep it consistent with linux unless we need something more in the future
objdump --disassemble --private-headers --reloc --dynamic-reloc --section-headers --syms --line-numbers --no-show-raw-insn --source --print-imm-hex "$l9_bin_file_unsanitized" > "$l9_tagged_path_unsanitized/$l9_bin_file_body_unsanitized.s"
else
exit 1
fi
fi
}
if [ -n "${CC}" ] && [ -n "${CXX}" ]; then
code_path="./shared/ebm_native"
tmp_path="./tmp/mk"
os_type=`uname`
# TODO: change this to accept lib_ebm_native_local.so or lib_ebm_native_local.dylib to allow for weird architectures build using sdists
if [ "$os_type" = "Linux" ]; then
final_binary="./python/interpret-core/interpret/lib/lib_ebm_native_linux_x64.so"
elif [ "$os_type" = "Darwin" ]; then
final_binary="./python/interpret-core/interpret/lib/lib_ebm_native_mac_x64.dylib"
else
printf "%s\n" "OS $os_type not recognized. We support clang/clang++ on macOS and gcc/g++ on Linux"
exit 1
fi
mkdir ./python
mkdir ./python/interpret-core
mkdir ./python/interpret-core/interpret
mkdir ./python/interpret-core/interpret/lib
extras="-DEBM_NATIVE_EXPORTS -DNDEBUG -I$code_path/inc -I$code_path/common_c -I$code_path/common_cpp -I$code_path/bridge_c -I$code_path/bridge_cpp -I$code_path -I$code_path/compute -I$code_path/compute/loss_functions -I$code_path/compute/metrics"
mkdir ./tmp
mkdir ./tmp/mk
mkdir ./staging
printf "Building from environment specified compiler\n"
printf "%s\n" "CC=${CC}"
printf "%s\n" "CXX=${CXX}"
printf "%s\n" "CPPFLAGS=${CPPFLAGS}"
printf "%s\n" "CFLAGS=${CFLAGS}"
printf "%s\n" "CXXFLAGS=${CXXFLAGS}"
printf "%s\n" "LDFLAGS=${LDFLAGS}"
printf "%s\n" "LOADLIBES=${LOADLIBES}"
printf "%s\n" "LDLIBS=${LDLIBS}"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/ApplyTermUpdate.cpp" -o "$tmp_path/ApplyTermUpdate.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/ApplyUpdate.cpp" -o "$tmp_path/ApplyUpdate.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/BinSumsBoosting.cpp" -o "$tmp_path/BinSumsBoosting.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/BinSumsInteraction.cpp" -o "$tmp_path/BinSumsInteraction.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/BoosterCore.cpp" -o "$tmp_path/BoosterCore.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/BoosterShell.cpp" -o "$tmp_path/BoosterShell.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/CalcInteractionStrength.cpp" -o "$tmp_path/CalcInteractionStrength.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/CutQuantile.cpp" -o "$tmp_path/CutQuantile.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/CutUniform.cpp" -o "$tmp_path/CutUniform.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/CutWinsorized.cpp" -o "$tmp_path/CutWinsorized.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/dataset_shared.cpp" -o "$tmp_path/dataset_shared.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/DataSetBoosting.cpp" -o "$tmp_path/DataSetBoosting.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/DataSetInteraction.cpp" -o "$tmp_path/DataSetInteraction.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/debug_ebm.cpp" -o "$tmp_path/debug_ebm.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/Discretize.cpp" -o "$tmp_path/Discretize.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/Term.cpp" -o "$tmp_path/Term.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/GenerateTermUpdate.cpp" -o "$tmp_path/GenerateTermUpdate.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/InitializeGradientsAndHessians.cpp" -o "$tmp_path/InitializeGradientsAndHessians.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/InteractionCore.cpp" -o "$tmp_path/InteractionCore.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/InteractionShell.cpp" -o "$tmp_path/InteractionShell.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/interpretable_numerics.cpp" -o "$tmp_path/interpretable_numerics.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/PartitionOneDimensionalBoosting.cpp" -o "$tmp_path/PartitionOneDimensionalBoosting.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/PartitionRandomBoosting.cpp" -o "$tmp_path/PartitionRandomBoosting.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/PartitionTwoDimensionalBoosting.cpp" -o "$tmp_path/PartitionTwoDimensionalBoosting.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/PartitionTwoDimensionalInteraction.cpp" -o "$tmp_path/PartitionTwoDimensionalInteraction.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/RandomDeterministic.cpp" -o "$tmp_path/RandomDeterministic.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/random.cpp" -o "$tmp_path/random.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/sampling.cpp" -o "$tmp_path/sampling.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/InnerBag.cpp" -o "$tmp_path/InnerBag.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/Tensor.cpp" -o "$tmp_path/Tensor.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/TensorTotalsBuild.cpp" -o "$tmp_path/TensorTotalsBuild.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/compute/Loss.cpp" -o "$tmp_path/Loss.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/compute/Registration.cpp" -o "$tmp_path/Registration.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/compute/zoned_bridge_c_functions.cpp" -o "$tmp_path/zoned_bridge_c_functions.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/compute/cpu_ebm/cpu_32.cpp" -o "$tmp_path/cpu_32.o"
${CXX} -c ${CPPFLAGS} ${CXXFLAGS} ${extras} -DZONE_main "$code_path/compute/cpu_ebm/cpu_64.cpp" -o "$tmp_path/cpu_64.o"
${CC} -c ${CPPFLAGS} ${CFLAGS} ${extras} -DZONE_main "$code_path/common_c/common_c.c" -o "$tmp_path/common_c.o"
${CC} -c ${CPPFLAGS} ${CFLAGS} ${extras} -DZONE_main "$code_path/common_c/logging.c" -o "$tmp_path/logging.o"
${CXX} ${LDFLAGS} -shared \
"$tmp_path/ApplyTermUpdate.o" \
"$tmp_path/ApplyUpdate.o" \
"$tmp_path/BinSumsBoosting.o" \
"$tmp_path/BinSumsInteraction.o" \
"$tmp_path/BoosterCore.o" \
"$tmp_path/BoosterShell.o" \
"$tmp_path/CalcInteractionStrength.o" \
"$tmp_path/CutQuantile.o" \
"$tmp_path/CutUniform.o" \
"$tmp_path/CutWinsorized.o" \
"$tmp_path/dataset_shared.o" \
"$tmp_path/DataSetBoosting.o" \
"$tmp_path/DataSetInteraction.o" \
"$tmp_path/debug_ebm.o" \
"$tmp_path/Discretize.o" \
"$tmp_path/Term.o" \
"$tmp_path/GenerateTermUpdate.o" \
"$tmp_path/InitializeGradientsAndHessians.o" \
"$tmp_path/InteractionCore.o" \
"$tmp_path/InteractionShell.o" \
"$tmp_path/interpretable_numerics.o" \
"$tmp_path/PartitionOneDimensionalBoosting.o" \
"$tmp_path/PartitionRandomBoosting.o" \
"$tmp_path/PartitionTwoDimensionalBoosting.o" \
"$tmp_path/PartitionTwoDimensionalInteraction.o" \
"$tmp_path/RandomDeterministic.o" \
"$tmp_path/random.o" \
"$tmp_path/sampling.o" \
"$tmp_path/InnerBag.o" \
"$tmp_path/Tensor.o" \
"$tmp_path/TensorTotalsBuild.o" \
"$tmp_path/Loss.o" \
"$tmp_path/Registration.o" \
"$tmp_path/zoned_bridge_c_functions.o" \
"$tmp_path/cpu_32.o" \
"$tmp_path/cpu_64.o" \
"$tmp_path/common_c.o" \
"$tmp_path/logging.o" \
${LOADLIBES} ${LDLIBS} -o "$final_binary"
exit 0
fi
release_64=1
debug_64=1
release_32=0
debug_32=0
is_asm=0
is_extra_debugging=0
for arg in "$@"; do
if [ "$arg" = "-no_release_64" ]; then
release_64=0
fi
if [ "$arg" = "-no_debug_64" ]; then
debug_64=0
fi
if [ "$arg" = "-release_32" ]; then
release_32=1
fi
if [ "$arg" = "-debug_32" ]; then
debug_32=1
fi
if [ "$arg" = "-asm" ]; then
is_asm=1
fi
if [ "$arg" = "-extra_debugging" ]; then
is_extra_debugging=1
fi
done
# TODO: this could be improved upon. There is no perfect solution AFAIK for getting the script directory, and I'm not too sure how the CDPATH thing works
# Look at BASH_SOURCE[0] as well and possibly select either it or $0
# The output here needs to not be the empty string for glob substitution below:
script_path_initial=`dirname -- "$0"`
# the space after the '= ' character is required
script_path_unsanitized=`CDPATH= cd -- "$script_path_initial" && pwd -P`
if [ ! -f "$script_path_unsanitized/build.sh" ] ; then
# there are all kinds of reasons why we might not have gotten the script path in $0. It's more of a convention
# than a requirement to have either the full path or even the script itself. There are far more complicated
# scripts out there that attempt to use various shell specific workarounds, like BASH_SOURCE[0] to best solve
# the problem, but it's possible in theory to be running over an SSL connection without a script on the local
# system at all, so getting the directory is a fundamentally unsolved problem. We can terminate though if
# we find ourselves in such a weird condition. This also happens when the "source" command is used.
printf "Could not find script file root directory for building InterpretML. Exiting."
exit 1
fi
root_path_unsanitized="$script_path_unsanitized"
tmp_path_unsanitized="$root_path_unsanitized/tmp"
python_lib_unsanitized="$root_path_unsanitized/python/interpret-core/interpret/lib"
staging_path_unsanitized="$root_path_unsanitized/staging"
src_path_unsanitized="$root_path_unsanitized/shared/ebm_native"
src_path_sanitized=`sanitize "$src_path_unsanitized"`
# a good referenece on writing shared libraries is at: https://akkadia.org/drepper/dsohowto.pdf
# re-enable these warnings when they are better supported by g++ or clang: -Wduplicated-cond -Wduplicated-branches -Wrestrict
both_args=""
both_args="$both_args -Wall -Wextra"
both_args="$both_args -Wunused-result"
both_args="$both_args -Wno-parentheses"
both_args="$both_args -Wdouble-promotion"
both_args="$both_args -Wshadow"
both_args="$both_args -Wformat=2"
both_args="$both_args -fvisibility=hidden"
both_args="$both_args -fno-math-errno -fno-trapping-math"
# TODO: once we have highly efficient tightly looped code, try no -fpic and see if that makes better code. The compiler can save a register in this case. See https://akkadia.org/drepper/dsohowto.pdf
# TODO: check no-plt compiler option
both_args="$both_args -fpic"
both_args="$both_args -pthread"
both_args="$both_args -DEBM_NATIVE_EXPORTS"
if [ $is_extra_debugging -ne 0 ]; then
both_args="$both_args -g"
fi
c_args="-std=c99 -Wstrict-prototypes"
cpp_args="-std=c++11"
cpp_args="$cpp_args -Wold-style-cast"
cpp_args="$cpp_args -fvisibility-inlines-hidden"
common_args="-I$src_path_sanitized/inc"
common_args="$common_args -I$src_path_sanitized/common_c"
common_args="$common_args -I$src_path_sanitized/common_cpp"
bridge_args="$common_args"
bridge_args="$bridge_args -I$src_path_sanitized/bridge_c"
bridge_args="$bridge_args -I$src_path_sanitized/bridge_cpp"
main_args="$bridge_args"
main_args="$main_args -I$src_path_sanitized"
compute_args="$bridge_args"
compute_args="$compute_args -I$src_path_sanitized/compute"
compute_args="$compute_args -I$src_path_sanitized/compute/loss_functions"
compute_args="$compute_args -I$src_path_sanitized/compute/metrics"
# add any other non-include options
common_args="$common_args -Wno-format-nonliteral"
link_args=""
os_type=`uname`
if [ "$os_type" = "Linux" ]; then
c_compiler=gcc
cpp_compiler=g++
# try moving some of these g++ specific warnings into both_args if clang eventually supports them
both_args="$both_args -Wlogical-op"
both_args="$both_args -march=core2"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args="$link_args -Wl,--version-script=$src_path_sanitized/ebm_native_exports.txt"
link_args="$link_args -Wl,--exclude-libs,ALL"
link_args="$link_args -Wl,-z,relro,-z,now"
link_args="$link_args -Wl,-O2"
link_args="$link_args -Wl,--sort-common"
link_args="$link_args -static-libgcc"
link_args="$link_args -static-libstdc++"
link_args="$link_args -shared"
printf "%s\n" "Creating initial directories"
[ -d "$staging_path_unsanitized" ] || mkdir -p "$staging_path_unsanitized"
ret_code=$?
if [ $ret_code -ne 0 ]; then
exit $ret_code
fi
[ -d "$python_lib_unsanitized" ] || mkdir -p "$python_lib_unsanitized"
ret_code=$?
if [ $ret_code -ne 0 ]; then
exit $ret_code
fi
if [ $release_64 -eq 1 ]; then
########################## Linux release|x64
printf "%s\n" "Compiling ebm_native with $c_compiler/$cpp_compiler for Linux release|x64"
obj_path_unsanitized="$tmp_path_unsanitized/gcc/obj/release/linux/x64/ebm_native"
bin_path_unsanitized="$tmp_path_unsanitized/gcc/bin/release/linux/x64/ebm_native"
bin_file="lib_ebm_native_linux_x64.so"
g_log_file_unsanitized="$obj_path_unsanitized/ebm_native_release_linux_x64_build_log.txt"
both_args_extra="-m64 -DNDEBUG -O3 -Wl,--wrap=memcpy -Wl,--wrap=exp -Wl,--wrap=log -Wl,--wrap=log2,--wrap=pow"
c_args_specific="$c_args $both_args $both_args_extra"
cpp_args_specific="$cpp_args $both_args $both_args_extra"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args_specific="$link_args $cpp_args_specific"
g_all_object_files_sanitized=""
g_compile_out_full=""
make_initial_paths_simple "$obj_path_unsanitized" "$bin_path_unsanitized"
compile_directory_c "$c_compiler" "$c_args_specific $common_args" "$src_path_unsanitized/common_c" "$obj_path_unsanitized" "$is_asm" "C"
compile_directory_c "$c_compiler" "$c_args_specific $bridge_args" "$src_path_unsanitized/bridge_c" "$obj_path_unsanitized" "$is_asm" "C"
compile_directory_cpp "$cpp_compiler" "$cpp_args_specific $main_args -DZONE_main" "$src_path_unsanitized" "$obj_path_unsanitized" "$is_asm" "main"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" "$is_asm" "cpu"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" "$is_asm" "avx512"
compile_file "$cpp_compiler" "$cpp_args_specific" "$src_path_unsanitized"/special/linux_wrap_functions.cpp "$obj_path_unsanitized" "$is_asm" "NONE"
link_file "$cpp_compiler" "$link_args_specific" "$bin_path_unsanitized" "$bin_file"
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
copy_bin_files "$bin_path_unsanitized" "$bin_file" "$python_lib_unsanitized" "$staging_path_unsanitized"
copy_asm_files "$obj_path_unsanitized" "$tmp_path_unsanitized" "$staging_path_unsanitized/$bin_file" "asm_release_64" "$is_asm"
fi
if [ $debug_64 -eq 1 ]; then
########################## Linux debug|x64
printf "%s\n" "Compiling ebm_native with $c_compiler/$cpp_compiler for Linux debug|x64"
obj_path_unsanitized="$tmp_path_unsanitized/gcc/obj/debug/linux/x64/ebm_native"
bin_path_unsanitized="$tmp_path_unsanitized/gcc/bin/debug/linux/x64/ebm_native"
bin_file="lib_ebm_native_linux_x64_debug.so"
g_log_file_unsanitized="$obj_path_unsanitized/ebm_native_debug_linux_x64_build_log.txt"
both_args_extra="-m64 -O1 -Wl,--wrap=memcpy -Wl,--wrap=exp -Wl,--wrap=log -Wl,--wrap=log2,--wrap=pow"
c_args_specific="$c_args $both_args $both_args_extra"
cpp_args_specific="$cpp_args $both_args $both_args_extra"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args_specific="$link_args $cpp_args_specific"
g_all_object_files_sanitized=""
g_compile_out_full=""
make_initial_paths_simple "$obj_path_unsanitized" "$bin_path_unsanitized"
compile_directory_c "$c_compiler" "$c_args_specific $common_args" "$src_path_unsanitized/common_c" "$obj_path_unsanitized" 0 "C"
compile_directory_c "$c_compiler" "$c_args_specific $bridge_args" "$src_path_unsanitized/bridge_c" "$obj_path_unsanitized" 0 "C"
compile_directory_cpp "$cpp_compiler" "$cpp_args_specific $main_args -DZONE_main" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "main"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "cpu"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "avx512"
compile_file "$cpp_compiler" "$cpp_args_specific" "$src_path_unsanitized"/special/linux_wrap_functions.cpp "$obj_path_unsanitized" 0 "NONE"
link_file "$cpp_compiler" "$link_args_specific" "$bin_path_unsanitized" "$bin_file"
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
copy_bin_files "$bin_path_unsanitized" "$bin_file" "$python_lib_unsanitized" "$staging_path_unsanitized"
fi
if [ $release_32 -eq 1 ]; then
########################## Linux release|x86
printf "%s\n" "Compiling ebm_native with $c_compiler/$cpp_compiler for Linux release|x86"
obj_path_unsanitized="$tmp_path_unsanitized/gcc/obj/release/linux/x86/ebm_native"
bin_path_unsanitized="$tmp_path_unsanitized/gcc/bin/release/linux/x86/ebm_native"
bin_file="lib_ebm_native_linux_x86.so"
g_log_file_unsanitized="$obj_path_unsanitized/ebm_native_release_linux_x86_build_log.txt"
both_args_extra="-msse2 -mfpmath=sse -m32 -DNDEBUG -O3"
c_args_specific="$c_args $both_args $both_args_extra"
cpp_args_specific="$cpp_args $both_args $both_args_extra"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args_specific="$link_args $cpp_args_specific"
g_all_object_files_sanitized=""
g_compile_out_full=""
make_initial_paths_simple "$obj_path_unsanitized" "$bin_path_unsanitized"
check_install "$tmp_path_unsanitized" "g++-multilib"
compile_directory_c "$c_compiler" "$c_args_specific $common_args" "$src_path_unsanitized/common_c" "$obj_path_unsanitized" 0 "C"
compile_directory_c "$c_compiler" "$c_args_specific $bridge_args" "$src_path_unsanitized/bridge_c" "$obj_path_unsanitized" 0 "C"
compile_directory_cpp "$cpp_compiler" "$cpp_args_specific $main_args -DZONE_main" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "main"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "cpu"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "avx512"
compile_file "$cpp_compiler" "$cpp_args_specific" "$src_path_unsanitized"/special/linux_wrap_functions.cpp "$obj_path_unsanitized" 0 "NONE"
link_file "$cpp_compiler" "$link_args_specific" "$bin_path_unsanitized" "$bin_file"
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
copy_bin_files "$bin_path_unsanitized" "$bin_file" "$python_lib_unsanitized" "$staging_path_unsanitized"
fi
if [ $debug_32 -eq 1 ]; then
########################## Linux debug|x86
printf "%s\n" "Compiling ebm_native with $c_compiler/$cpp_compiler for Linux debug|x86"
obj_path_unsanitized="$tmp_path_unsanitized/gcc/obj/debug/linux/x86/ebm_native"
bin_path_unsanitized="$tmp_path_unsanitized/gcc/bin/debug/linux/x86/ebm_native"
bin_file="lib_ebm_native_linux_x86_debug.so"
g_log_file_unsanitized="$obj_path_unsanitized/ebm_native_debug_linux_x86_build_log.txt"
both_args_extra="-msse2 -mfpmath=sse -m32 -O1"
c_args_specific="$c_args $both_args $both_args_extra"
cpp_args_specific="$cpp_args $both_args $both_args_extra"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args_specific="$link_args $cpp_args_specific"
g_all_object_files_sanitized=""
g_compile_out_full=""
make_initial_paths_simple "$obj_path_unsanitized" "$bin_path_unsanitized"
check_install "$tmp_path_unsanitized" "g++-multilib"
compile_directory_c "$c_compiler" "$c_args_specific $common_args" "$src_path_unsanitized/common_c" "$obj_path_unsanitized" 0 "C"
compile_directory_c "$c_compiler" "$c_args_specific $bridge_args" "$src_path_unsanitized/bridge_c" "$obj_path_unsanitized" 0 "C"
compile_directory_cpp "$cpp_compiler" "$cpp_args_specific $main_args -DZONE_main" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "main"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "cpu"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "avx512"
compile_file "$cpp_compiler" "$cpp_args_specific" "$src_path_unsanitized"/special/linux_wrap_functions.cpp "$obj_path_unsanitized" 0 "NONE"
link_file "$cpp_compiler" "$link_args_specific" "$bin_path_unsanitized" "$bin_file"
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
copy_bin_files "$bin_path_unsanitized" "$bin_file" "$python_lib_unsanitized" "$staging_path_unsanitized"
fi
elif [ "$os_type" = "Darwin" ]; then
# reference on rpath & install_name: https://www.mikeash.com/pyblog/friday-qa-2009-11-06-linking-and-install-names.html
# TODO: make these real options instead of forcing them to the same as x64
release_arm=$release_64
debug_arm=$debug_64
# try moving some of these clang specific warnings into both_args if g++ eventually supports them
c_compiler=clang
cpp_compiler=clang++
both_args="$both_args -Wnull-dereference"
both_args="$both_args -Wgnu-zero-variadic-macro-arguments"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args="$link_args -dynamiclib"
printf "%s\n" "Creating initial directories"
[ -d "$staging_path_unsanitized" ] || mkdir -p "$staging_path_unsanitized"
ret_code=$?
if [ $ret_code -ne 0 ]; then
exit $ret_code
fi
[ -d "$python_lib_unsanitized" ] || mkdir -p "$python_lib_unsanitized"
ret_code=$?
if [ $ret_code -ne 0 ]; then
exit $ret_code
fi
if [ $release_64 -eq 1 ]; then
########################## macOS release|x64
printf "%s\n" "Compiling ebm_native with $c_compiler/$cpp_compiler for macOS release|x64"
obj_path_unsanitized="$tmp_path_unsanitized/clang/obj/release/mac/x64/ebm_native"
bin_path_unsanitized="$tmp_path_unsanitized/clang/bin/release/mac/x64/ebm_native"
bin_file="lib_ebm_native_mac_x64.dylib"
g_log_file_unsanitized="$obj_path_unsanitized/ebm_native_release_mac_x64_build_log.txt"
both_args_extra="-march=core2 -target x86_64-apple-macos10.12 -m64 -DNDEBUG -O3"
c_args_specific="$c_args $both_args $both_args_extra"
cpp_args_specific="$cpp_args $both_args $both_args_extra"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args_specific="-install_name @rpath/$bin_file $link_args $cpp_args_specific"
g_all_object_files_sanitized=""
g_compile_out_full=""
make_initial_paths_simple "$obj_path_unsanitized" "$bin_path_unsanitized"
compile_directory_c "$c_compiler" "$c_args_specific $common_args" "$src_path_unsanitized/common_c" "$obj_path_unsanitized" "$is_asm" "C"
compile_directory_c "$c_compiler" "$c_args_specific $bridge_args" "$src_path_unsanitized/bridge_c" "$obj_path_unsanitized" "$is_asm" "C"
compile_directory_cpp "$cpp_compiler" "$cpp_args_specific $main_args -DZONE_main" "$src_path_unsanitized" "$obj_path_unsanitized" "$is_asm" "main"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" "$is_asm" "cpu"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" "$is_asm" "avx512"
link_file "$cpp_compiler" "$link_args_specific" "$bin_path_unsanitized" "$bin_file"
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
copy_bin_files "$bin_path_unsanitized" "$bin_file" "$python_lib_unsanitized" "$staging_path_unsanitized"
copy_asm_files "$obj_path_unsanitized" "$tmp_path_unsanitized" "$staging_path_unsanitized/$bin_file" "asm_release_64" "$is_asm"
fi
if [ $debug_64 -eq 1 ]; then
########################## macOS debug|x64
printf "%s\n" "Compiling ebm_native with $c_compiler/$cpp_compiler for macOS debug|x64"
obj_path_unsanitized="$tmp_path_unsanitized/clang/obj/debug/mac/x64/ebm_native"
bin_path_unsanitized="$tmp_path_unsanitized/clang/bin/debug/mac/x64/ebm_native"
bin_file="lib_ebm_native_mac_x64_debug.dylib"
g_log_file_unsanitized="$obj_path_unsanitized/ebm_native_debug_mac_x64_build_log.txt"
both_args_extra="-march=core2 -target x86_64-apple-macos10.12 -m64 -O1 -fsanitize=address,undefined -fno-sanitize-recover=address,undefined -fno-optimize-sibling-calls -fno-omit-frame-pointer"
c_args_specific="$c_args $both_args $both_args_extra"
cpp_args_specific="$cpp_args $both_args $both_args_extra"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args_specific="-install_name @rpath/$bin_file $link_args $cpp_args_specific"
g_all_object_files_sanitized=""
g_compile_out_full=""
make_initial_paths_simple "$obj_path_unsanitized" "$bin_path_unsanitized"
compile_directory_c "$c_compiler" "$c_args_specific $common_args" "$src_path_unsanitized/common_c" "$obj_path_unsanitized" 0 "C"
compile_directory_c "$c_compiler" "$c_args_specific $bridge_args" "$src_path_unsanitized/bridge_c" "$obj_path_unsanitized" 0 "C"
compile_directory_cpp "$cpp_compiler" "$cpp_args_specific $main_args -DZONE_main" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "main"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "cpu"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "avx512"
link_file "$cpp_compiler" "$link_args_specific" "$bin_path_unsanitized" "$bin_file"
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
copy_bin_files "$bin_path_unsanitized" "$bin_file" "$python_lib_unsanitized" "$staging_path_unsanitized"
fi
if [ $release_arm -eq 1 ]; then
########################## macOS release|arm
printf "%s\n" "Compiling ebm_native with $c_compiler/$cpp_compiler for macOS release|arm"
obj_path_unsanitized="$tmp_path_unsanitized/clang/obj/release/mac/arm/ebm_native"
bin_path_unsanitized="$tmp_path_unsanitized/clang/bin/release/mac/arm/ebm_native"
bin_file="lib_ebm_native_mac_arm.dylib"
g_log_file_unsanitized="$obj_path_unsanitized/ebm_native_release_mac_arm_build_log.txt"
both_args_extra="-target arm64-apple-macos11 -m64 -DNDEBUG -O3"
c_args_specific="$c_args $both_args $both_args_extra"
cpp_args_specific="$cpp_args $both_args $both_args_extra"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args_specific="-install_name @rpath/$bin_file $link_args $cpp_args_specific"
g_all_object_files_sanitized=""
g_compile_out_full=""
make_initial_paths_simple "$obj_path_unsanitized" "$bin_path_unsanitized"
compile_directory_c "$c_compiler" "$c_args_specific $common_args" "$src_path_unsanitized/common_c" "$obj_path_unsanitized" "$is_asm" "C"
compile_directory_c "$c_compiler" "$c_args_specific $bridge_args" "$src_path_unsanitized/bridge_c" "$obj_path_unsanitized" "$is_asm" "C"
compile_directory_cpp "$cpp_compiler" "$cpp_args_specific $main_args -DZONE_main" "$src_path_unsanitized" "$obj_path_unsanitized" "$is_asm" "main"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" "$is_asm" "cpu"
link_file "$cpp_compiler" "$link_args_specific" "$bin_path_unsanitized" "$bin_file"
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
copy_bin_files "$bin_path_unsanitized" "$bin_file" "$python_lib_unsanitized" "$staging_path_unsanitized"
copy_asm_files "$obj_path_unsanitized" "$tmp_path_unsanitized" "$staging_path_unsanitized/$bin_file" "asm_release_arm" "$is_asm"
fi
if [ $debug_arm -eq 1 ]; then
########################## macOS debug|arm
printf "%s\n" "Compiling ebm_native with $c_compiler/$cpp_compiler for macOS debug|arm"
obj_path_unsanitized="$tmp_path_unsanitized/clang/obj/debug/mac/arm/ebm_native"
bin_path_unsanitized="$tmp_path_unsanitized/clang/bin/debug/mac/arm/ebm_native"
bin_file="lib_ebm_native_mac_arm_debug.dylib"
g_log_file_unsanitized="$obj_path_unsanitized/ebm_native_debug_mac_arm_build_log.txt"
both_args_extra="-target arm64-apple-macos11 -m64 -O1 -fsanitize=address,undefined -fno-sanitize-recover=address,undefined -fno-optimize-sibling-calls -fno-omit-frame-pointer"
c_args_specific="$c_args $both_args $both_args_extra"
cpp_args_specific="$cpp_args $both_args $both_args_extra"
# the linker wants to have the most dependent .o/.so/.dylib files listed FIRST
link_args_specific="-install_name @rpath/$bin_file $link_args $cpp_args_specific"
g_all_object_files_sanitized=""
g_compile_out_full=""
make_initial_paths_simple "$obj_path_unsanitized" "$bin_path_unsanitized"
compile_directory_c "$c_compiler" "$c_args_specific $common_args" "$src_path_unsanitized/common_c" "$obj_path_unsanitized" 0 "C"
compile_directory_c "$c_compiler" "$c_args_specific $bridge_args" "$src_path_unsanitized/bridge_c" "$obj_path_unsanitized" 0 "C"
compile_directory_cpp "$cpp_compiler" "$cpp_args_specific $main_args -DZONE_main" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "main"
compile_compute "$cpp_compiler" "$cpp_args_specific $compute_args" "$src_path_sanitized" "$src_path_unsanitized" "$obj_path_unsanitized" 0 "cpu"
link_file "$cpp_compiler" "$link_args_specific" "$bin_path_unsanitized" "$bin_file"
printf "%s\n" "$g_compile_out_full"
printf "%s\n" "$g_compile_out_full" > "$g_log_file_unsanitized"
copy_bin_files "$bin_path_unsanitized" "$bin_file" "$python_lib_unsanitized" "$staging_path_unsanitized"
fi
else
printf "%s\n" "OS $os_type not recognized. We support clang/clang++ on macOS and gcc/g++ on Linux"
exit 1
fi