# Copyright (C) 2015-2018 UCSC Computational Genomics Lab # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. define help Supported targets: prepare, develop, docs, sdist, clean, test, pypi, docker and push_docker. Please note that all build targets require a virtualenv to be active. The 'prepare' target installs Toil's build requirements into the current virtualenv. The 'develop' target creates an editable install of Toil and its runtime requirements in the current virtualenv. The install is called 'editable' because changes to the source code immediately affect the virtualenv. Set the 'extras' variable to ensure that the 'develop' target installs support for extras; some tests require extras to be installed. Consult setup.py for the list of supported extras. To install Toil in develop mode with all extras, run make develop extras=[all] The 'sdist' target creates a source distribution of Toil. It is used for some unit tests and for installing the currently checked out version of Toil into the appliance image. The 'clean' target cleans up the side effects of 'develop', 'sdist', 'docs', 'pypi' and 'docker' on this machine. It does not undo externally visible effects like removing packages already uploaded to PyPI. The 'docs' target uses Sphinx to create HTML documentation in the docs/_build directory Targets are provided to run Toil's tests. Note that these targets do *not* automatically install Toil's dependencies; it is recommended to 'make develop' before running any of them. The 'test' target runs Toil's unit tests serially with pytest. It will run some docker tests and setup. If you wish to avoid this, use the 'test_offline' target instead. Note: this target does not capture output from the terminal. For any of the test targets, set the 'tests' variable to run a particular test, e.g. make test tests=src/toil/test/sort/sortTest.py::SortTest::testSort The 'test_offline' target is similar to 'test' but it skips the docker dependent tests and their setup. It can also be used to invoke individual tests, e.g. make test_offline tests_local=src/toil/test/sort/sortTest.py::SortTest::testSort The 'integration_test_local' target runs toil's integration tests. These are more thorough but also more costly than the regular unit tests. For the AWS integration tests to run, the environment variable 'TOIL_AWS_KEYNAME' must be set. This user will be charged for expenses acrued during the test. This test does not capture terminal output. The 'integration_test' target is the same as the previous except that it does capture output. The 'pypi' target publishes the current commit of Toil to PyPI after enforcing that the working copy and the index are clean. The 'docker' target builds the Docker images that make up the Toil appliance. You may set the TOIL_DOCKER_REGISTRY variable to override the default registry that the 'push_docker' target pushes the appliance images to, for example: TOIL_DOCKER_REGISTRY=quay.io/USER make docker If Docker is not installed, Docker-related targets tasks and tests will be skipped. The same can be achieved by setting TOIL_DOCKER_REGISTRY to an empty string. The 'push_docker' target pushes the Toil appliance images to a remote Docker registry. It requires the TOIL_DOCKER_REGISTRY variable to be set to a value other than the default to avoid accidentally pushing to the official Docker registry for Toil. The TOIL_DOCKER_NAME environment variable can be set to customize the appliance image name that is created by the 'docker' target and pushed by the 'push_docker' target. The Toil team\'s continuous integration system overrides this variable to avoid conflicts between concurrently executing builds for the same revision, e.g. toil-pr and toil-it. endef export help help: @printf "$$help" # This Makefile uses bash features like printf and <() SHELL=bash python=python pip=pip tests=src tests_local=src/toil/test # do slightly less than travis timeout of 10 min. pytest_args_local=-vv --timeout=530 extras= dist_version:=$(shell $(python) version_template.py distVersion) sdist_name:=toil-$(dist_version).tar.gz docker_tag:=$(shell $(python) version_template.py dockerTag) default_docker_registry:=$(shell $(python) version_template.py dockerRegistry) docker_path:=$(strip $(shell which docker)) export TOIL_DOCKER_REGISTRY?=$(default_docker_registry) export TOIL_DOCKER_NAME?=$(shell $(python) version_template.py dockerName) export TOIL_APPLIANCE_SELF:=$(TOIL_DOCKER_REGISTRY)/$(TOIL_DOCKER_NAME):$(docker_tag) ifndef BUILD_NUMBER green=\033[0;32m normal=\033[0m red=\033[0;31m cyan=\033[0;36m endif develop: check_venv $(pip) install -e .$(extras) clean_develop: check_venv - $(pip) uninstall -y toil - rm -rf src/*.egg-info - rm src/toil/version.py sdist: dist/$(sdist_name) dist/$(sdist_name): check_venv @test -f dist/$(sdist_name) && mv dist/$(sdist_name) dist/$(sdist_name).old || true $(python) setup.py sdist @test -f dist/$(sdist_name).old \ && ( cmp -s <(tar -xOzf dist/$(sdist_name)) <(tar -xOzf dist/$(sdist_name).old) \ && mv dist/$(sdist_name).old dist/$(sdist_name) \ && printf "$(cyan)No significant changes to sdist, reinstating backup.$(normal)\n" \ || rm dist/$(sdist_name).old ) \ || true clean_sdist: - rm -rf dist - rm src/toil/version.py # We always claim to be Travis, so that local test runs will not skip Travis tests. # Gitlab doesn't run tests via the Makefile. # This target will skip building docker and all docker based tests test_offline: check_venv check_build_reqs @printf "$(cyan)All docker related tests will be skipped.$(normal)\n" TOIL_SKIP_DOCKER=True \ TRAVIS=true \ $(python) -m pytest $(pytest_args_local) $(tests_local) # The auto-deployment test needs the docker appliance test: check_venv check_build_reqs docker TRAVIS=true \ $(python) -m pytest --cov=toil $(pytest_args_local) $(tests) # For running integration tests locally in series (uses the -s argument for pyTest) integration_test_local: check_venv check_build_reqs sdist push_docker TRAVIS=true \ $(python) run_tests.py --local integration-test $(tests) test_integration: check_venv check_build_reqs docker TRAVIS=true \ $(python) run_tests.py integration-test $(tests) ifdef TOIL_DOCKER_REGISTRY docker_image:=$(TOIL_DOCKER_REGISTRY)/$(TOIL_DOCKER_NAME) docker_short_tag:=$(shell $(python) version_template.py dockerShortTag) docker_minimal_tag:=$(shell $(python) version_template.py dockerMinimalTag) grafana_image:=$(TOIL_DOCKER_REGISTRY)/toil-grafana prometheus_image:=$(TOIL_DOCKER_REGISTRY)/toil-prometheus mtail_image:=$(TOIL_DOCKER_REGISTRY)/toil-mtail define tag_docker @printf "$(cyan)Removing old tag $2. This may fail but that's expected.$(normal)\n" -docker rmi $2 docker tag $1 $2 @printf "$(green)Tagged appliance image $1 as $2.$(normal)\n" endef docker: docker/Dockerfile @set -ex \ ; cd docker \ ; docker build --tag=$(docker_image):$(docker_tag) -f Dockerfile . @set -ex \ ; cd dashboard/prometheus \ ; docker build --tag=$(prometheus_image):$(docker_tag) -f Dockerfile . @set -ex \ ; cd dashboard/grafana \ ; docker build --tag=$(grafana_image):$(docker_tag) -f Dockerfile . @set -ex \ ; cd dashboard/mtail \ ; docker build --tag=$(mtail_image):$(docker_tag) -f Dockerfile . ifdef BUILD_NUMBER $(call tag_docker,$(docker_image):$(docker_tag),$(docker_image):$(docker_short_tag)) $(call tag_docker,$(docker_image):$(docker_tag),$(docker_image):$(docker_minimal_tag)) endif docker/$(sdist_name): dist/$(sdist_name) cp $< $@ docker/Dockerfile: docker/Dockerfile.py docker/$(sdist_name) _TOIL_SDIST_NAME=$(sdist_name) $(python) docker/Dockerfile.py > $@ clean_docker: -rm docker/Dockerfile docker/$(sdist_name) -docker rmi $(docker_image):$(docker_tag) push_docker: docker check_docker_registry for i in $$(seq 1 5); do docker push $(docker_image):$(docker_tag) && break || sleep 60; done for i in $$(seq 1 5); do docker push $(grafana_image):$(docker_tag) && break || sleep 60; done for i in $$(seq 1 5); do docker push $(prometheus_image):$(docker_tag) && break || sleep 60; done for i in $$(seq 1 5); do docker push $(mtail_image):$(docker_tag) && break || sleep 60; done else docker docker_push clean_docker: @printf "$(cyan)Skipping '$@' target as TOIL_DOCKER_REGISTRY is empty or Docker is not installed.$(normal)\n" endif docs: check_venv check_build_reqs # Strange, but seemingly benign Sphinx warning floods stderr if not filtered: cd docs && make html clean_docs: check_venv - cd docs && make clean clean: clean_develop clean_sdist clean_pypi clean_docs check_build_reqs: @$(python) -c 'import mock; import pytest' \ || ( printf "$(red)Build requirements are missing. Run 'make prepare' to install them.$(normal)\n" ; false ) prepare: check_venv $(pip) install mock==1.0.1 pytest==4.3.1 pytest-cov==2.6.1 stubserver==1.0.1 pytest-timeout==1.3.3 cwltest check_venv: @$(python) -c 'import sys, os; sys.exit( int( 0 if "VIRTUAL_ENV" in os.environ else 1 ) )' \ || ( printf "$(red)A virtualenv must be active.$(normal)\n" ; false ) check_clean_working_copy: @printf "$(green)Checking if your working copy is clean ...$(normal)\n" @git diff --exit-code > /dev/null \ || ( printf "$(red)Your working copy looks dirty.$(normal)\n" ; false ) @git diff --cached --exit-code > /dev/null \ || ( printf "$(red)Your index looks dirty.$(normal)\n" ; false ) @test -z "$$(git ls-files --other --exclude-standard --directory)" \ || ( printf "$(red)You have untracked files:$(normal)\n" \ ; git ls-files --other --exclude-standard --directory \ ; false ) check_docker_registry: @test "$(default_docker_registry)" != "$(TOIL_DOCKER_REGISTRY)" || test -n "$$BUILD_NUMBER" \ || ( printf '$(red)Please set TOIL_DOCKER_REGISTRY to a value other than \ $(default_docker_registry) and ensure that you have permissions to push \ to that registry. Only CI builds should push to $(default_docker_registry).$(normal)\n' ; false ) check_cpickle: # fail if cPickle.dump(s) called without HIGHEST_PROTOCOL # https://github.com/BD2KGenomics/toil/issues/1503 ! find src -iname '*.py' | xargs grep 'cPickle.dump' | grep --invert-match HIGHEST_PROTOCOL .PHONY: help \ prepare \ check_cpickle \ develop clean_develop \ sdist clean_sdist \ test test_offline test_parallel integration_test \ pypi clean_pypi \ docs clean_docs \ clean \ check_venv \ check_clean_working_copy \ check_running_on_jenkins \ check_build_reqs \ docker clean_docker push_docker