Revision 56e08548600099b570c894fcdd672183c4115976 authored by Ian Harry on 26 October 2020, 15:56:35 UTC, committed by GitHub on 26 October 2020, 15:56:35 UTC
1 parent 0182f62
pycbc_make_html_page
#!/usr/bin/env python
# Copyright (C) 2015 Christopher M. Biwer
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Generals
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import argparse
import os, stat
import pycbc.results
import random
import shutil
import zipfile
import codecs
from ligo import segments
from jinja2 import Environment, FileSystemLoader
from pycbc.results.render import get_embedded_config, render_workflow_html_template, setup_template_render
from pycbc.workflow import segment
import pycbc.version
def examine_dir(cwd):
"""
Looks in a directory and returns all subdirs and files. If there is a
zipped file, it will unzip it, and return the extracted files as well.
"""
# list everything in this directory and loop over them
names = os.listdir(cwd)
dirs, nondirs = [], []
for name in names:
# if it is a directory append the dir list
if os.path.isdir(os.path.join(cwd, name)):
dirs.append(name)
# check if it is a zip file, unzip it, and add files to list
elif zipfile.is_zipfile(name):
zip = zipfile.ZipFile(name)
zip.extractall()
# loop over extracted files
for extracted_name in zip.namelist():
# check if extracted a dir
if os.path.isdir(os.path.join(cwd, extracted_name)):
dirs.append(os.path.dirname(extracted_name))
# check if extracted a file into this current working dir
elif os.path.exists(os.path.join(cwd, os.path.basename(extracted_name))):
nondirs.append(os.path.basename(extracted_name))
else:
nondirs.append(name)
return cwd, dirs, nondirs
def render_sitemap_page(output_path, dirs):
"""
Creates sitemap page.
"""
dirs.sort(key=lambda x: x.path)
render_workflow_html_template(output_path, 'sitemap.html', dirs)
class Directory():
"""
Class used to relate all sub-directories and files in a directory.
"""
# a list for all instances of this class
instances = []
def __init__(self, path, plots_dir):
# find all subdirs and filenames in directory
cwd, subdirs, filenames = examine_dir(path)
# save subdirs
self.path = path.replace(plots_dir, '')
self.subdirs = [Directory(path+'/'+subdir, plots_dir) for subdir in subdirs]
# loop over all filenames
self.files = []
for filename in filenames:
# check if this is a configuration file for a file
extension = filename.split('.')[-1]
config_filename = filename.replace(extension, 'file.ini')
if filename.endswith('file.ini'):
continue
# check if configuration file exists for file
elif not os.path.exists(config_filename):
self.config_filename = None
# append file to directory
self.files.append(File(plots_dir+'/'+self.path+'/'+filename,
plots_dir+'/'+self.path+'/'+config_filename))
# append to class list
self.instances.append(self)
def name(self):
"""
Returns the name of the directory.
"""
return self.path.split('/')[-1]
def title(self):
"""
Returns a string of the directory name with underscores as spaces
and capslock.
"""
return self.path.split('/')[-1].replace('_', ' ').title()
def level(self):
"""
Counts how far into the output filesystem this directory is.
"""
return self.path.count('/')
class File():
"""
Class used to keep track of files.
"""
def __init__(self, path, config_path):
# save paths
self.path = path
self.config_path = config_path
def filename(self):
"""
Returns filename of File as a string.
"""
return self.path.split('/')[-1]
def render(self):
"""
Renders a template for the File using the configuration file if present.
"""
return setup_template_render(self.path, self.config_path)
default_logo_location = "https://raw.githubusercontent.com/gwastro/" + \
"pycbc-logo/master/pycbc_logo_name.png"
# parse command line
parser = argparse.ArgumentParser(usage='pycbc_make_html_page \
[--options]',
description="Create static html pages of a filesystem's content.")
parser.add_argument("--version", action="version",
version=pycbc.version.git_verbose_msg)
parser.add_argument('-f', '--template-file', type=str,
help='Template file to use for skeleton html page.')
parser.add_argument('-b', '--output-path', type=str,
help='Path on web server for main html page.')
parser.add_argument('-p', '--plots-dir', type=str,
help='Path to the directory that contains plots.')
parser.add_argument('-t', '--analysis-title', type=str,
help='Title to include at the top of each page.',
default="")
parser.add_argument('-s', '--analysis-subtitle', type=str,
help='Subtitle to include at the top of each page.',
default="")
parser.add_argument('-l', '--logo', type=str,
default=default_logo_location,
help='File location of logo to include at top of page')
parser.add_argument('-v', '--verbose', action='store_true',
help='Print extra debugging information.', default=False)
parser.add_argument('-P', '--protect-results', action='store_true',
help='Make the output web pages read only.', default=False)
opts = parser.parse_args()
# edit command line options
analysis_title = opts.analysis_title.strip('"').rstrip('"')
analysis_subtitle = opts.analysis_subtitle.strip('"').rstrip('"')
if opts.template_file[0] != '/':
full_template_path = pycbc.results.__path__[0] + '/' + opts.template_file
else:
full_template_path = opts.template_file
input_template = full_template_path.split('/')[-1]
input_path = full_template_path.rstrip(input_template)
# setup template
env = Environment(loader=FileSystemLoader(input_path))
env.globals.update(setup_template_render=setup_template_render)
env.globals.update(get_embedded_config=get_embedded_config)
env.globals.update(path_exists=os.path.exists)
env.globals.update(list=list)
template = env.get_template(input_template)
# find all subdirs and the top-level subdirs
Directory(opts.plots_dir, opts.plots_dir)
dirs = [cwd for cwd in Directory.instances]
dirs_0 = [cwd for cwd in Directory.instances if cwd.path.count('/') == 1]
# sort alphanumerically
# FIXME: could move this into Directory when subdirs and files are appended
dirs.sort(key=lambda x: x.title())
for cwd in dirs:
cwd.subdirs.sort(key=lambda x: x.title())
cwd.files.sort(key=lambda x: x.path)
dirs_0.sort(key=lambda x: x.title())
local_logo = opts.logo[:4] != "http"
if local_logo:
logo_filename = os.path.basename(opts.logo)
shutil.copy2(opts.logo, opts.output_path)
# loop over all directories
for cwd in dirs:
# render template
context = {'analysis_title' : analysis_title,
'analysis_subtitle' : analysis_subtitle,
'dirs_0' : dirs_0,
'dir' : cwd,
'dot_dot_str' : cwd.level() * '../',
'plots_dir' : opts.plots_dir}
if local_logo:
context['logo'] = context['dot_dot_str'] + logo_filename
else:
context['logo'] = opts.logo
output = template.render(context)
# if directory does not exist make it and copy directory permissions
if not os.path.exists(opts.output_path+cwd.path):
os.makedirs(opts.output_path+cwd.path)
shutil.copymode(opts.plots_dir+cwd.path, opts.output_path+'/'+cwd.path)
# save html page
with codecs.open(opts.output_path+cwd.path+'/index.html', "w",
encoding='utf-8') as fp:
fp.write(output)
# copy all files to html directory
for cwd in dirs:
for file in cwd.files:
try:
shutil.copy2(file.path, opts.output_path+'/'+cwd.path+'/'+file.filename())
except IOError:
pass
# make sitemap page
sitemap_dir = '/sitemap'
if not os.path.exists(opts.plots_dir+sitemap_dir):
os.makedirs(opts.plots_dir+sitemap_dir)
render_sitemap_page(opts.plots_dir+sitemap_dir+'/well.html', dirs)
cwd = Directory(opts.plots_dir+sitemap_dir, opts.plots_dir)
context = {'analysis_title' : analysis_title,
'analysis_subtitle' : analysis_subtitle,
'dirs_0' : dirs_0,
'dir' : cwd,
'dot_dot_str' : cwd.level() * '../',
'plots_dir' : opts.plots_dir}
if local_logo:
context['logo'] = context['dot_dot_str'] + logo_filename
else:
context['logo'] = opts.logo
output = template.render(context)
if not os.path.exists(opts.output_path+cwd.path):
os.makedirs(opts.output_path+cwd.path)
with open(opts.output_path+cwd.path+'/index.html', "w") as fp:
fp.write(output)
# copy css, js, and font files to html directory
cssDir = pycbc.results.__path__[0] + '/static/css/'
jsDir = pycbc.results.__path__[0] + '/static/js/'
fontsDir = pycbc.results.__path__[0] + '/static/fonts/'
cssOutputDir = opts.output_path + '/static/css/'
jsOutputDir = opts.output_path + '/static/js/'
fontsOutputDir = opts.output_path + '/static/fonts/'
if not os.path.exists(cssOutputDir):
shutil.copytree(cssDir, cssOutputDir)
if not os.path.exists(jsOutputDir):
shutil.copytree(jsDir, jsOutputDir)
if not os.path.exists(fontsOutputDir):
shutil.copytree(fontsDir, fontsOutputDir)
# When bundling assets into pyinstaller binaries the static directory is
# created with restrictions that are too restrictive. So, go through and make
# sure that directories are readable+executable and files are readable.
for dirpath, dirnames, filenames in os.walk(opts.output_path + '/static'):
os.chmod(dirpath, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
for filename in filenames:
path = os.path.join(dirpath, filename)
os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
# Protect the results if the requested
if opts.protect_results:
for dirpath, dirnames, filenames in os.walk(opts.output_path):
# Do not open the box
dir_permission = stat.S_IREAD | stat.S_IEXEC
existing_perm = os.stat(dirpath)
if bool(existing_perm.st_mode & stat.S_IRGRP):
dir_permission = dir_permission | stat.S_IRGRP | stat.S_IXGRP
if bool(existing_perm.st_mode & stat.S_IROTH):
dir_permission = dir_permission | stat.S_IROTH | stat.S_IXOTH
os.chmod(dirpath, dir_permission)
for filename in filenames:
path = os.path.join(dirpath, filename)
os.chmod(path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
Computing file changes ...