https://github.com/catboost/catboost
Tip revision: 0db6c27bc7187212124d908edd23b751fab26eff authored by Evgueni Petrov on 17 May 2022, 05:32:27 UTC
Release 1.0.6
Release 1.0.6
Tip revision: 0db6c27
ya
#!/usr/bin/env python
import os
import sys
import platform
import json
URLS = ["https://storage.mds.yandex.net/get-devtools-opensource/373962/000ced60f88958175457814f345562b9"]
MD5 = "000ced60f88958175457814f345562b9"
RETRIES = 5
HASH_PREFIX = 10
HOME_DIR = os.path.expanduser('~')
def create_dirs(path):
try:
os.makedirs(path)
except OSError as e:
import errno
if e.errno != errno.EEXIST:
raise
return path
def misc_root():
return create_dirs(os.getenv('YA_CACHE_DIR') or os.path.join(HOME_DIR, '.ya'))
def tool_root():
return create_dirs(os.getenv('YA_CACHE_DIR_TOOLS') or os.path.join(misc_root(), 'tools'))
def ya_token():
def get_token_from_file():
try:
with open(os.environ.get('YA_TOKEN_PATH', os.path.join(HOME_DIR, '.ya_token')), 'r') as f:
return f.read().strip()
except:
pass
return os.getenv('YA_TOKEN') or get_token_from_file()
TOOLS_DIR = tool_root()
def uniq(size=6):
import string
import random
return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(size))
def _fetch(url, into):
import hashlib
try:
from urllib2 import urlopen
from urllib2 import Request
from urlparse import urlparse
except ImportError:
from urllib.request import urlopen
from urllib.request import Request
from urllib.parse import urlparse
request = Request(str(url))
request.add_header('User-Agent', 'ya-bootstrap')
if urlparse(url).netloc == 'proxy.sandbox.yandex-team.ru':
token = ya_token()
if token:
request.add_header('Authorization', 'OAuth {}'.format(token))
md5 = hashlib.md5()
sys.stderr.write('Downloading %s ' % url)
sys.stderr.flush()
conn = urlopen(request, timeout=10)
sys.stderr.write('[')
sys.stderr.flush()
try:
with open(into, 'wb') as f:
while True:
block = conn.read(1024 * 1024)
sys.stderr.write('.')
sys.stderr.flush()
if block:
md5.update(block)
f.write(block)
else:
break
return md5.hexdigest()
finally:
sys.stderr.write('] ')
sys.stderr.flush()
def _atomic_fetch(url, into, md5):
tmp_dest = into + '.' + uniq()
try:
real_md5 = _fetch(url, tmp_dest)
if real_md5 != md5:
raise Exception('MD5 mismatched: %s differs from %s' % (real_md5, md5))
os.rename(tmp_dest, into)
sys.stderr.write('OK\n')
except Exception as e:
sys.stderr.write('ERROR: ' + str(e) + '\n')
raise
finally:
try:
os.remove(tmp_dest)
except OSError:
pass
def _extract(path, into):
import tarfile
tar = tarfile.open(path, errorlevel=2)
tar.extractall(path=into)
tar.close()
def _get(urls, md5):
dest_path = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX])
if not os.path.exists(dest_path):
for iter in range(RETRIES):
try:
_atomic_fetch(urls[iter % len(urls)], dest_path, md5)
break
except Exception:
if iter + 1 == RETRIES:
raise
else:
import time
time.sleep(iter)
return dest_path
def _get_dir(urls, md5, ya_name):
dest_dir = os.path.join(TOOLS_DIR, md5[:HASH_PREFIX] + '_d')
if os.path.isfile(os.path.join(dest_dir, ya_name)):
return dest_dir
try:
packed_path = _get(urls, md5)
except Exception:
if os.path.isfile(os.path.join(dest_dir, ya_name)):
return dest_dir
raise
tmp_dir = dest_dir + '.' + uniq()
try:
try:
_extract(packed_path, tmp_dir)
except Exception:
if os.path.isfile(os.path.join(dest_dir, ya_name)):
return dest_dir
raise
try:
os.rename(tmp_dir, dest_dir)
except OSError as e:
import errno
if e.errno != errno.ENOTEMPTY:
raise
return dest_dir
finally:
import shutil
shutil.rmtree(tmp_dir, ignore_errors=True)
try:
os.remove(packed_path)
except Exception:
pass
def _mine_arc_root():
return os.path.dirname(os.path.realpath(__file__))
def main():
if not os.path.exists(TOOLS_DIR):
os.makedirs(TOOLS_DIR)
with open(_get(URLS, MD5), 'r') as fp:
meta = json.load(fp)['data']
my_platform = platform.system().lower()
my_machine = platform.machine().lower()
if my_platform == 'linux':
my_platform = 'linux-ppc64le' if 'ppc64le' in platform.platform() else 'linux_musl'
if my_platform == 'darwin' and my_machine == 'arm64':
my_platform = 'darwin-arm64'
# match by max prefix length, prefer shortest
best_key = max(meta.keys(), key=lambda x: (len(os.path.commonprefix([my_platform, x])), -len(x)))
value = meta[best_key]
if len(sys.argv) == 2 and sys.argv[1].startswith('--print-sandbox-id='):
target = sys.argv[1].split('=')[1]
best_target = max(meta.keys(), key=lambda x: len(os.path.commonprefix([target, x])))
sys.stdout.write(str(meta[best_target]['resource_id']) + '\n')
exit(0)
ya_name = {'win32': 'ya-bin.exe'}.get(best_key, 'ya-bin') # XXX
ya_dir = _get_dir(value['urls'], value['md5'], ya_name)
# Popen `args` must have `str` type
ya_path = str(os.path.join(ya_dir, ya_name))
env = os.environ.copy()
if 'YA_SOURCE_ROOT' not in env:
src_root = _mine_arc_root()
if src_root is not None:
env['YA_SOURCE_ROOT'] = src_root
# For more info see YT-14105
if 'LD_PRELOAD' in env:
sys.stderr.write("Warn: LD_PRELOAD='{}' is specified and may affect the correct operation of the ya\n".format(env['LD_PRELOAD']))
if os.name == 'nt':
import subprocess
p = subprocess.Popen([ya_path] + sys.argv[1:], env=env)
p.wait()
sys.exit(p.returncode)
else:
os.execve(ya_path, [ya_path] + sys.argv[1:], env)
if __name__ == '__main__':
try:
main()
except Exception as e:
sys.stderr.write('ERROR: ' + str(e) + '\n')
sys.exit(1)