https://github.com/jvivian/one_off_scripts
Raw File
Tip revision: 3ad04be99cd01e6a047c1b530cc8a1de82bd862e authored by John Vivian on 02 February 2017, 01:31:18 UTC
Refactor SRA pipeline to use faster method than fastq-dump
Tip revision: 3ad04be
toil-scripts-jenkins.py
import os
import subprocess
import textwrap


def setup():
    """Calls setup script which installs pipeline dependencies"""
    subprocess.check_call(['/usr/bin/bash', '-ex'], stdin=textwrap.dedent("""
        virtualenv s3am
        . s3am/bin/activate
        pip install --pre s3am
        deactivate
        # Create Toil venv
        virtualenv toil
        . venv/bin/activate
        pip install pytest toil boto
        deactivate
        # Expose binaries to the PATH
        mkdir bin
        export PATH=$PATH:${PWD}/bin
        ln -snf ${PWD}/s3am/bin/s3am bin/
        ln -snf ${PWD}/toil/bin/toil bin/
        ln -snf ${PWD}/toil/bin/boto bin/
        ln -snf ${PWD}/toil/bin/pytest bin/
        # Set PYTHONPATH
        export PYTHONPATH=$(python -c 'from os.path import abspath as a;import sys;print a("src")' $0)"""))


def determine_modified_subdirs():
    """Determines directories modified since previous commit"""
    out = subprocess.check_output(['git', 'diff-tree', '--name-only', 'HEAD', '-r']).split('\n')
    return {os.path.dirname(x)for x in out if 'src' in x}


def contains_test(path):
    """If directory contains a 'test' subdir, returns non-empty set"""
    return {x for x in next(os.walk(path))[1] if 'test' in x}


def main():
    setup()
    modified_subdirs = determine_modified_subdirs()
    subdirs_to_test = filter(contains_test, modified_subdirs)
    for i, subdir in enumerate(subdirs_to_test):
        subprocess.check_call('py.test', subdir, '--doctest-modules', '--junitxml=test-report-{}.xml'.format(i))


if __name__ == '__main__':
    main()
back to top