https://github.com/jvivian/one_off_scripts
Raw File
Tip revision: 3ad04be99cd01e6a047c1b530cc8a1de82bd862e authored by John Vivian on 02 February 2017, 01:31:18 UTC
Refactor SRA pipeline to use faster method than fastq-dump
Tip revision: 3ad04be
generate_ssec_url.py
#!/usr/bin/env python2.7
"""
Original Author: Hannes Schmidt
Author: John Vivian

Date: 2-25-16

Generate URLs that mimic requester pays with encryption
"""
import base64
import hashlib

with open('master.key', 'r') as f:
    master_key = f.read()


def curl_command( key, expires=600, location=None ):
    """
    Returns a curl command that downloads an encrypted object in 
    
    :param Key key: a Boto Key object representing the key for which to genrate a URL
    :param expires: number of seconds for which the result can be used
    :param str location: the location (slightly different to region) of the key's bucket. 
           If None, a location will be determined automatically which will slow this 
           function down considerably. 
    """
    bucket = key.bucket
    if location is None:
        location = bucket.get_location( )
    if location:
        base_url = 'https://s3-' + location + '.amazonaws.com'
    else:
        base_url = 'https://s3.amazonaws.com'
    url = '/'.join( [ base_url, bucket.name, key.name ] )
    sse_key = hashlib.sha256( master_key + str( url ) ).digest( )
    assert len( sse_key ) == 32
    encoded_sse_key = base64.b64encode( sse_key )
    encoded_sse_key_md5 = base64.b64encode( hashlib.md5( sse_key ).digest( ) )
    prefix = 'x-amz-server-side-encryption-customer-'
    headers = {
        prefix + 'algorithm': 'AES256',
        prefix + 'key': encoded_sse_key,
        prefix + 'key-md5': encoded_sse_key_md5 }
    signature_headers = {
        prefix + 'algorithm': 'AES256',
        'x-amz-request-payer': 'requester'
    }
    return ' '.join( [ 'curl',
                         '--create-dirs',
                         '--continue-at', '-',
                         '--output', key.name,
                         "'" + key.generate_url( expires, headers=signature_headers ) + "'" ] + [
                         "-H '%s:%s'" % (k, v) for k, v in headers.iteritems( ) ] )
back to top