gen_gallery.py
# -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
Sphinx-Gallery Generator
========================
Attaches Sphinx-Gallery to Sphinx in order to generate the galleries
when building the documentation.
"""
from __future__ import division, print_function, absolute_import
import codecs
import copy
import re
import os
from . import glr_path_static
from . import sphinx_compatibility
from .gen_rst import generate_dir_rst, SPHX_GLR_SIG
from .docs_resolv import embed_code_links
from .downloads import generate_zipfiles
try:
FileNotFoundError
except NameError:
# Python2
FileNotFoundError = IOError
DEFAULT_GALLERY_CONF = {
'filename_pattern': re.escape(os.sep) + 'plot',
'examples_dirs': os.path.join('..', 'examples'),
'subsection_order': None,
'gallery_dirs': 'auto_examples',
'backreferences_dir': None,
'doc_module': (),
'reference_url': {},
# build options
'plot_gallery': True,
'download_all_examples': True,
'abort_on_example_error': False,
'failing_examples': {},
'expected_failing_examples': set(),
}
logger = sphinx_compatibility.getLogger('sphinx-gallery')
def clean_gallery_out(build_dir):
"""Deletes images under the sphx_glr namespace in the build directory"""
# Sphinx hack: sphinx copies generated images to the build directory
# each time the docs are made. If the desired image name already
# exists, it appends a digit to prevent overwrites. The problem is,
# the directory is never cleared. This means that each time you build
# the docs, the number of images in the directory grows.
#
# This question has been asked on the sphinx development list, but there
# was no response: https://git.net/ml/sphinx-dev/2011-02/msg00123.html
#
# The following is a hack that prevents this behavior by clearing the
# image build directory from gallery images each time the docs are built.
# If sphinx changes their layout between versions, this will not
# work (though it should probably not cause a crash).
# Tested successfully on Sphinx 1.0.7
build_image_dir = os.path.join(build_dir, '_images')
if os.path.exists(build_image_dir):
filelist = os.listdir(build_image_dir)
for filename in filelist:
if filename.startswith('sphx_glr') and filename.endswith('png'):
os.remove(os.path.join(build_image_dir, filename))
def parse_config(app):
"""Process the Sphinx Gallery configuration"""
try:
plot_gallery = eval(app.builder.config.plot_gallery)
except TypeError:
plot_gallery = bool(app.builder.config.plot_gallery)
gallery_conf = copy.deepcopy(DEFAULT_GALLERY_CONF)
gallery_conf.update(app.config.sphinx_gallery_conf)
gallery_conf.update(plot_gallery=plot_gallery)
gallery_conf.update(
abort_on_example_error=app.builder.config.abort_on_example_error)
gallery_conf['src_dir'] = app.builder.srcdir
backreferences_warning = """\n========
Sphinx-Gallery now requires you to set the configuration variable
'backreferences_dir' in your config to activate the
backreferences. That is mini galleries clustered by the functions used
in the example scripts. Have a look at it in sphinx-gallery
https://sphinx-gallery.readthedocs.io/en/stable/index.html#examples-using-numpy-linspace
"""
if gallery_conf.get("mod_example_dir", False):
update_msg = """\nFor a quick fix try replacing 'mod_example_dir'
by 'backreferences_dir' in your conf.py file. If that does not solve the
present issue read carefully how to update in the online documentation
https://sphinx-gallery.readthedocs.io/en/latest/advanced_configuration.html#references-to-examples"""
gallery_conf['backreferences_dir'] = gallery_conf['mod_example_dir']
logger.warning(
"Old configuration for backreferences detected \n"
"using the configuration variable `mod_example_dir`\n"
"%s%s",
backreferences_warning,
update_msg,
type=DeprecationWarning)
elif gallery_conf['backreferences_dir'] is None:
no_care_msg = """
If you don't care about this features set in your conf.py
'backreferences_dir': False\n"""
logger.warning(backreferences_warning + no_care_msg)
gallery_conf['backreferences_dir'] = os.path.join(
'modules', 'generated')
logger.warning(
"Using old default 'backreferences_dir':'%s'.\n"
"This will be disabled in future releases\n",
gallery_conf['backreferences_dir'],
type=DeprecationWarning)
# this assures I can call the config in other places
app.config.sphinx_gallery_conf = gallery_conf
app.config.html_static_path.append(glr_path_static())
return gallery_conf
def get_subsections(srcdir, examples_dir, sortkey):
"""Returns the list of subsections of a gallery
Parameters
----------
srcdir : str
absolute path to directory containing conf.py
examples_dir : str
path to the examples directory relative to conf.py
sortkey : sortkey
Returns
-------
out : list
sorted list of gallery subsection folder names
"""
subfolders = [subfolder for subfolder in os.listdir(examples_dir)
if os.path.exists(os.path.join(examples_dir, subfolder, 'README.txt'))]
base_examples_dir_path = os.path.relpath(examples_dir, srcdir)
subfolders_with_path = [os.path.join(base_examples_dir_path, item)
for item in subfolders]
sorted_subfolders = sorted(subfolders_with_path, key=sortkey)
return [subfolders[i] for i in [subfolders_with_path.index(item)
for item in sorted_subfolders]]
def _prepare_sphx_glr_dirs(gallery_conf, srcdir):
"""Creates necessary folders for sphinx_gallery files """
examples_dirs = gallery_conf['examples_dirs']
gallery_dirs = gallery_conf['gallery_dirs']
if not isinstance(examples_dirs, list):
examples_dirs = [examples_dirs]
if not isinstance(gallery_dirs, list):
gallery_dirs = [gallery_dirs]
for outdir in gallery_dirs:
if not os.path.exists(outdir):
os.makedirs(outdir)
if bool(gallery_conf['backreferences_dir']):
backreferences_dir = os.path.join(
srcdir, gallery_conf['backreferences_dir'])
if not os.path.exists(backreferences_dir):
os.makedirs(backreferences_dir)
return zip(examples_dirs, gallery_dirs)
def generate_gallery_rst(app):
"""Generate the Main examples gallery reStructuredText
Start the sphinx-gallery configuration and recursively scan the examples
directories in order to populate the examples gallery
"""
logger.info('Generating gallery...', color='white')
gallery_conf = parse_config(app)
clean_gallery_out(app.builder.outdir)
seen_backrefs = set()
computation_times = []
workdirs = _prepare_sphx_glr_dirs(gallery_conf,
app.builder.srcdir)
for examples_dir, gallery_dir in workdirs:
examples_dir = os.path.join(app.builder.srcdir, examples_dir)
gallery_dir = os.path.join(app.builder.srcdir, gallery_dir)
if not os.path.exists(os.path.join(examples_dir, 'README.txt')):
raise FileNotFoundError("Main example directory {0} does not "
"have a README.txt file. Please write "
"one to introduce your gallery."
.format(examples_dir))
# Here we don't use an os.walk, but we recurse only twice: flat is
# better than nested.
this_fhindex, this_computation_times = generate_dir_rst(
examples_dir, gallery_dir, gallery_conf, seen_backrefs)
computation_times += this_computation_times
# we create an index.rst with all examples
with codecs.open(os.path.join(gallery_dir, 'index.rst'), 'w',
encoding='utf-8') as fhindex:
# :orphan: to suppress "not included in TOCTREE" sphinx warnings
fhindex.write(":orphan:\n\n" + this_fhindex)
for subsection in get_subsections(app.builder.srcdir, examples_dir, gallery_conf['subsection_order']):
src_dir = os.path.join(examples_dir, subsection)
target_dir = os.path.join(gallery_dir, subsection)
this_fhindex, this_computation_times = generate_dir_rst(src_dir, target_dir, gallery_conf,
seen_backrefs)
fhindex.write(this_fhindex)
computation_times += this_computation_times
if gallery_conf['download_all_examples']:
download_fhindex = generate_zipfiles(gallery_dir)
fhindex.write(download_fhindex)
fhindex.write(SPHX_GLR_SIG)
if gallery_conf['plot_gallery']:
logger.info("Computation time summary:", color='white')
for time_elapsed, fname in sorted(computation_times)[::-1]:
if time_elapsed is not None:
logger.info("\t- %s : %.2g sec", fname, time_elapsed)
else:
logger.info("\t- %s : not run", fname)
def touch_empty_backreferences(app, what, name, obj, options, lines):
"""Generate empty back-reference example files
This avoids inclusion errors/warnings if there are no gallery
examples for a class / module that is being parsed by autodoc"""
if not bool(app.config.sphinx_gallery_conf['backreferences_dir']):
return
examples_path = os.path.join(app.srcdir,
app.config.sphinx_gallery_conf[
"backreferences_dir"],
"%s.examples" % name)
if not os.path.exists(examples_path):
# touch file
open(examples_path, 'w').close()
def sumarize_failing_examples(app, exception):
"""Collects the list of falling examples during build and prints them with the traceback
Raises ValueError if there where failing examples
"""
if exception is not None:
return
# Under no-plot Examples are not run so nothing to summarize
if not app.config.sphinx_gallery_conf['plot_gallery']:
return
gallery_conf = app.config.sphinx_gallery_conf
failing_examples = set(gallery_conf['failing_examples'].keys())
expected_failing_examples = set([os.path.normpath(os.path.join(app.srcdir, path))
for path in
gallery_conf['expected_failing_examples']])
examples_expected_to_fail = failing_examples.intersection(
expected_failing_examples)
if examples_expected_to_fail:
logger.info("Examples failing as expected:", color='brown')
for fail_example in examples_expected_to_fail:
logger.info('%s failed leaving traceback:', fail_example)
logger.info(gallery_conf['failing_examples'][fail_example])
examples_not_expected_to_fail = failing_examples.difference(
expected_failing_examples)
fail_msgs = []
if examples_not_expected_to_fail:
fail_msgs.append("Unexpected failing examples:")
for fail_example in examples_not_expected_to_fail:
fail_msgs.append(fail_example + ' failed leaving traceback:\n' +
gallery_conf['failing_examples'][fail_example] + '\n')
examples_not_expected_to_pass = expected_failing_examples.difference(
failing_examples)
if examples_not_expected_to_pass:
fail_msgs.append("Examples expected to fail, but not failling:\n" +
"Please remove these examples from\n" +
"sphinx_gallery_conf['expected_failing_examples']\n" +
"in your conf.py file"
"\n".join(examples_not_expected_to_pass))
if fail_msgs:
raise ValueError("Here is a summary of the problems encountered when "
"running the examples\n\n" + "\n".join(fail_msgs) +
"\n" + "-" * 79)
def get_default_config_value(key):
def default_getter(conf):
return conf['sphinx_gallery_conf'].get(key, DEFAULT_GALLERY_CONF[key])
return default_getter
def setup(app):
"""Setup sphinx-gallery sphinx extension"""
sphinx_compatibility._app = app
app.add_config_value('sphinx_gallery_conf', DEFAULT_GALLERY_CONF, 'html')
for key in ['plot_gallery', 'abort_on_example_error']:
app.add_config_value(key, get_default_config_value(key), 'html')
app.add_stylesheet('gallery.css')
# Sphinx < 1.6 calls it `_extensions`, >= 1.6 is `extensions`.
extensions_attr = '_extensions' if hasattr(
app, '_extensions') else 'extensions'
if 'sphinx.ext.autodoc' in getattr(app, extensions_attr):
app.connect('autodoc-process-docstring', touch_empty_backreferences)
app.connect('builder-inited', generate_gallery_rst)
app.connect('build-finished', sumarize_failing_examples)
app.connect('build-finished', embed_code_links)
metadata = {'parallel_read_safe': True}
return metadata
def setup_module():
# HACK: Stop nosetests running setup() above
pass
