https://github.com/gwastro/pycbc
Revision b574af5aeb8e5b22c040f0e6cecd6796e6a5cb69 authored by Duncan Brown on 27 November 2018, 23:45:28 UTC, committed by GitHub on 27 November 2018, 23:45:28 UTC
1 parent b1a1759
Raw File
Tip revision: b574af5aeb8e5b22c040f0e6cecd6796e6a5cb69 authored by Duncan Brown on 27 November 2018, 23:45:28 UTC
Set for 1.13.2 release
Tip revision: b574af5
pycbc_make_html_page
#!/usr/bin/env python

# Copyright (C) 2015 Christopher M. Biwer
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Generals
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

import argparse
import os, stat
import pycbc.results
import random
import shutil
import zipfile
import codecs
from glue import segments
from jinja2 import Environment, FileSystemLoader
from pycbc.results.render import get_embedded_config, render_workflow_html_template, setup_template_render
from pycbc.workflow import segment
import pycbc.version

def examine_dir(cwd):
    """
    Looks in a directory and returns all subdirs and files. If there is a
    zipped file, it will unzip it, and return the extracted files as well.
    """

    # list everything in this directory and loop over them
    names = os.listdir(cwd)
    dirs, nondirs = [], []
    for name in names:

        # if it is a directory append the dir list
        if os.path.isdir(os.path.join(cwd, name)):
            dirs.append(name)

        # check if it is a zip file, unzip it, and add files to list
        elif zipfile.is_zipfile(name):
            zip = zipfile.ZipFile(name)
            zip.extractall()

            # loop over extracted files
            for extracted_name in zip.namelist():

                # check if extracted a dir
                if os.path.isdir(os.path.join(cwd, extracted_name)):
                    dirs.append(os.path.dirname(extracted_name))

                # check if extracted a file into this current working dir
                elif os.path.exists(os.path.join(cwd, os.path.basename(extracted_name))):
                    nondirs.append(os.path.basename(extracted_name))

        else:
            nondirs.append(name)

    return cwd, dirs, nondirs

def render_sitemap_page(output_path, dirs):
    """
    Creates sitemap page.
    """
    dirs.sort(key=lambda x: x.path)
    render_workflow_html_template(output_path, 'sitemap.html', dirs)

class Directory():
    """
    Class used to relate all sub-directories and files in a directory.
    """

    # a list for all instances of this class
    instances = []

    def __init__(self, path, plots_dir):

        # find all subdirs and filenames in directory
        cwd, subdirs, filenames = examine_dir(path)

        # save subdirs
        self.path    = path.replace(plots_dir, '')
        self.subdirs = [Directory(path+'/'+subdir, plots_dir) for subdir in subdirs]

        # loop over all filenames
        self.files = []
        for filename in filenames:

            # check if this is a configuration file for a file
            extension = filename.split('.')[-1]
            config_filename = filename.replace(extension, 'file.ini')
            if filename.endswith('file.ini'):
                continue

            # check if configuration file exists for file
            elif not os.path.exists(config_filename):
                self.config_filename = None

            # append file to directory
            self.files.append(File(plots_dir+'/'+self.path+'/'+filename,
                                   plots_dir+'/'+self.path+'/'+config_filename))

        # append to class list
        self.instances.append(self)

    def name(self):
        """
        Returns the name of the directory.
        """

        return self.path.split('/')[-1]

    def title(self):
        """
        Returns a string of the directory name with underscores as spaces
        and capslock.
        """

        return self.path.split('/')[-1].replace('_', ' ').title()

    def level(self):
        """
        Counts how far into the output filesystem this directory is.
        """

        return self.path.count('/')

class File():
    """
    Class used to keep track of files.
    """

    def __init__(self, path, config_path):

        # save paths
        self.path        = path
        self.config_path = config_path

    def filename(self):
        """
        Returns filename of File as a string.
        """

        return self.path.split('/')[-1]

    def render(self):
        """
        Renders a template for the File using the configuration file if present.
        """

        return setup_template_render(self.path, self.config_path)

# parse command line
parser = argparse.ArgumentParser(usage='pycbc_make_html_page \
[--options]',
                  description="Create static html pages of a filesystem's content.")
parser.add_argument("--version", action="version",
                  version=pycbc.version.git_verbose_msg)
parser.add_argument('-f', '--template-file', type=str,
                  help='Template file to use for skeleton html page.')
parser.add_argument('-b', '--output-path', type=str,
                  help='Path on web server for main html page.')
parser.add_argument('-p', '--plots-dir', type=str,
                  help='Path to the directory that contains plots.')
parser.add_argument('-t', '--analysis-title', type=str,
                  help='Title to include at the top of each page.',
                  default="")
parser.add_argument('-s', '--analysis-subtitle', type=str,
                  help='Subtitle to include at the top of each page.',
                  default="")
parser.add_argument('-v', '--verbose', action='store_true',
                  help='Print extra debugging information.', default=False)
parser.add_argument('-P', '--protect-results', action='store_true',
                  help='Make the output web pages read only.', default=False)
opts = parser.parse_args()

# edit command line options
analysis_title = opts.analysis_title.strip('"').rstrip('"')
analysis_subtitle = opts.analysis_subtitle.strip('"').rstrip('"')

if opts.template_file[0] != '/':
    full_template_path = pycbc.results.__path__[0] + '/' + opts.template_file
else:
    full_template_path = opts.template_file

input_template = full_template_path.split('/')[-1]
input_path = full_template_path.rstrip(input_template)

# setup template
env = Environment(loader=FileSystemLoader(input_path))
env.globals.update(setup_template_render=setup_template_render)
env.globals.update(get_embedded_config=get_embedded_config)
env.globals.update(path_exists=os.path.exists)
env.globals.update(list=list)
template = env.get_template(input_template)

# find all subdirs and the top-level subdirs
Directory(opts.plots_dir, opts.plots_dir)
dirs   = [cwd for cwd in Directory.instances]
dirs_0 = [cwd for cwd in Directory.instances if cwd.path.count('/') == 1]

# sort alphanumerically
# FIXME: could move this into Directory when subdirs and files are appended
dirs.sort(key=lambda x: x.title())
for cwd in dirs:
    cwd.subdirs.sort(key=lambda x: x.title())
    cwd.files.sort(key=lambda x: x.path)
dirs_0.sort(key=lambda x: x.title())

# loop over all directories
for cwd in dirs:

    # render template
    context = {'analysis_title'    : analysis_title,
               'analysis_subtitle' : analysis_subtitle,
               'dirs_0'            : dirs_0,
               'dir'               : cwd,
               'dot_dot_str'       : cwd.level() * '../',
               'plots_dir'         : opts.plots_dir}
    output = template.render(context)

    # if directory does not exist make it and copy directory permissions
    if not os.path.exists(opts.output_path+cwd.path):
        os.makedirs(opts.output_path+cwd.path)
        shutil.copymode(opts.plots_dir+cwd.path, opts.output_path+'/'+cwd.path)

    # save html page
    with codecs.open(opts.output_path+cwd.path+'/index.html', "w",
                     encoding='utf-8') as fp:
        fp.write(output)

# copy all files to html directory
for cwd in dirs:
    for file in cwd.files:
        try:
            shutil.copy2(file.path, opts.output_path+'/'+cwd.path+'/'+file.filename())
        except IOError:
            pass

# make sitemap page
sitemap_dir = '/sitemap'
if not os.path.exists(opts.plots_dir+sitemap_dir):
    os.makedirs(opts.plots_dir+sitemap_dir)
render_sitemap_page(opts.plots_dir+sitemap_dir+'/well.html', dirs)
cwd = Directory(opts.plots_dir+sitemap_dir, opts.plots_dir)
context = {'analysis_title'    : analysis_title,
           'analysis_subtitle' : analysis_subtitle,
           'dirs_0'            : dirs_0,
           'dir'               : cwd,
           'dot_dot_str'       : cwd.level() * '../',
           'plots_dir'         : opts.plots_dir}
output = template.render(context)
if not os.path.exists(opts.output_path+cwd.path):
    os.makedirs(opts.output_path+cwd.path)
with open(opts.output_path+cwd.path+'/index.html', "wb") as fp:
    fp.write(output)

# copy css, js, and font files to html directory
cssDir       = pycbc.results.__path__[0] + '/static/css/'
jsDir        = pycbc.results.__path__[0] + '/static/js/'
fontsDir      = pycbc.results.__path__[0] + '/static/fonts/'
cssOutputDir = opts.output_path + '/static/css/'
jsOutputDir  = opts.output_path + '/static/js/'
fontsOutputDir  = opts.output_path + '/static/fonts/'
if not os.path.exists(cssOutputDir):
    shutil.copytree(cssDir, cssOutputDir)
if not os.path.exists(jsOutputDir):
    shutil.copytree(jsDir, jsOutputDir)
if not os.path.exists(fontsOutputDir):
    shutil.copytree(fontsDir, fontsOutputDir)


# When bundling assets into pyinstaller binaries the static directory is
# created with restrictions that are too restrictive.  So, go through and make
# sure that directories are readable+executable and files are readable.

for dirpath, dirnames, filenames in os.walk(opts.output_path + '/static'):
    os.chmod(dirpath, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)

    for filename in filenames:
        path = os.path.join(dirpath, filename)
        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)

# Protect the results if the requested
if opts.protect_results:
    for dirpath, dirnames, filenames in os.walk(opts.output_path):

        # Do not open the box
        dir_permission = stat.S_IREAD | stat.S_IEXEC
        existing_perm = os.stat(dirpath)
        if bool(existing_perm.st_mode & stat.S_IRGRP):
            dir_permission = dir_permission | stat.S_IRGRP | stat.S_IXGRP
        if bool(existing_perm.st_mode & stat.S_IROTH):
            dir_permission = dir_permission | stat.S_IROTH | stat.S_IXOTH

        os.chmod(dirpath, dir_permission)

        for filename in filenames:
            path = os.path.join(dirpath, filename)
            os.chmod(path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
back to top