Revision 958055bb6e51acccd79d80e944d3bb19c31c0fcd authored by Alon Zakai on 26 October 2020, 16:26:38 UTC, committed by Alon Zakai on 26 October 2020, 16:26:38 UTC
1 parent f9d491b
get_wiki.py
#!/usr/bin/env python
# Copyright 2014 The Emscripten Authors. All rights reserved.
# Emscripten is available under two separate licenses, the MIT license and the
# University of Illinois/NCSA Open Source License. Both these licenses can be
# found in the LICENSE file.
#
# This script gets the Emscripten wiki, converts files from markdown to
# restructure text using pandoc (and also prepends the text with a title/heading
# based on the filename)
#
# It also fixes up inline code items to become links to api-reference
#
# It should be called prior to building the site.
#
from __future__ import print_function
import optparse
import os
import re
import shutil
import stat
import sys
import time
import api_items
wiki_repo = 'https://github.com/emscripten-core/emscripten.wiki.git'
output_dir = './wiki_static/'
logfilename = 'log-get-wiki.txt'
# GetmMap of code items from api_items.py
mapped_wiki_inline_code = api_items.get_mapped_items()
# Add any additional mapped items that seem reasonable.
mapped_wiki_inline_code['getValue(ptr, type)'] = ':js:func:`getValue(ptr, type) <getValue>`'
mapped_wiki_inline_code['setValue(ptr, value, type)'] = ':js:func:`setValue(ptr, value, type) <setValue>`'
wiki_checkout = 'emscripten.wiki/'
temp_set_of_codemarkup = set()
logfile = open(logfilename, 'w')
# snapshot_version_information = '.. note:: This is a **snapshot** of the wiki: %s\n\n' % strftime("%a, %d %b %Y %H:%M", gmtime())
snapshot_version_information = '.. note:: This article was migrated from the wiki (%s) and is now the "master copy" (the version in the wiki will be deleted). It may not be a perfect rendering of the original but we hope to fix that soon!\n\n' % time.strftime("%a, %d %b %Y %H:%M", time.gmtime())
def CleanWiki():
"""Delete the wiki clone directory and all contained files.
"""
def errorhandler(func, path, exc_info):
# where func is os.listdir, os.remove, or os.rmdir; path is the argument to that function that caused it to fail; and exc_info is a tuple returned by sys.exc_info()
print(func)
print(path)
print(exc_info)
os.chmod(path, stat.S_IWRITE)
os.remove(path)
try:
shutil.rmtree(output_dir, ignore_errors=False, onerror=errorhandler)
print('Old wiki clone removed')
except IOError:
print('No directory to clean found')
def CloneWiki():
"""
Clone the wiki into a temporary location (first cleaning)
"""
# Clean up existing repo
CleanWiki()
# Create directory for output and temporary files
try:
os.makedirs(output_dir)
print('Created directory')
except OSError:
pass
# Clone
git_clone_command = 'git clone %s %s' % (wiki_repo, wiki_checkout)
print(git_clone_command)
os.system(git_clone_command)
def ConvertFilesToRst():
"""
Add template to specified page object (wikitools)
"""
indexfiletext = '============================\nWiki snapshot (ready-for-review)\n============================\n\n%s\n.. toctree::\n :maxdepth: 2\n' % snapshot_version_information
for file in os.listdir(wiki_checkout):
if not file.endswith(".md"):
continue
inputfilename = wiki_checkout + file
markdown = open(inputfilename).read()
if 'This article has moved from the wiki to the new site' in markdown:
continue
if 'This page has been migrated to the main site' in markdown:
continue
print(file)
# get name of file
filenamestripped = os.path.splitext(file)[0]
indexfiletext += '\n %s' % filenamestripped
outputfilename = output_dir + filenamestripped + '.rst'
command = 'pandoc -f markdown -t rst -o "%s" "%s"' % (outputfilename, inputfilename)
print(command)
if os.system(command):
sys.exit(1)
title = filenamestripped.replace('-', ' ')
# print title
logfile.write('title from filename: %s \n' % title)
# add import message to title
title += ' (wiki-import)'
length = len(title)
# print length
headerbar = ''
for number in range(length):
headerbar += '='
page_reference = filenamestripped
page_reference_link_text = '.. _%s:\n\n' % page_reference
titlebar = page_reference_link_text + headerbar + '\n' + title + '\n' + headerbar + '\n'
textinfile = ''
# Add titlebar to start of the file (from the filename)
textinfile += titlebar
# Add wiki snapshot information
textinfile += snapshot_version_information
with open(outputfilename) as infile:
for line in infile:
textinfile += line
# print textinfile
with open(outputfilename, 'w') as outfile:
outfile.write(textinfile)
# write the index
with open(output_dir + 'index.rst', 'w') as outfile:
outfile.write(indexfiletext)
def FixupConvertedRstFiles():
"""Add template to specified page object (wikitools)
"""
def fixInternalWikiLinks(aOldText):
"""
Fixes wiki links in [[linkname]] format by changing this to a document link in current directory.
"""
def fixwikilinks(matchobj):
# print 'matcobj0: %s' % matchobj.group(0)
# print 'matcobj1: %s' % matchobj.group(1)
linktext = matchobj.group(1)
linktext = linktext.replace(' ', '-')
# linktext = ':doc:`%s`' % linktext
# use reference for linking as allows pages to be moved around
linktext = ':ref:`%s`' % linktext
# print 'linkdoc: %s' % linktext
logfile.write('linkdoc: %s \n' % linktext)
return linktext
# print 'fixing wiki links'
return re.sub(r'\[\[(.+?)\]\]', fixwikilinks, aOldText)
def fixWikiCodeMarkupToCodeLinks(aOldText):
"""
Links "known" code objects if they are found in wiki markup.
"""
def fixcodemarkuplinks(matchobj):
# print 'Inline code: %s' % matchobj.group(0)
# print 'matcobj1: %s' % matchobj.group(1)
temp_set_of_codemarkup.add(matchobj.group(0))
linktext = matchobj.group(1)
if linktext in mapped_wiki_inline_code:
logfile.write('Replace: %s \n' % mapped_wiki_inline_code[linktext])
return mapped_wiki_inline_code[linktext]
return matchobj.group(0) # linktext
# print 'fixing up code markup to code reference'
return re.sub(r'``(.+?)``', fixcodemarkuplinks, aOldText)
for file in os.listdir(output_dir):
if file.endswith(".rst"):
input_file = output_dir + file
# print input_file
textinfile = ''
with open(input_file) as infile:
for line in infile:
textinfile += line
# print textinfile
# fix up broken wiki-page links in files
textinfile = fixInternalWikiLinks(textinfile)
# convert codemarkup to links if possible
textinfile = fixWikiCodeMarkupToCodeLinks(textinfile)
with open(input_file, 'w') as outfile:
outfile.write(textinfile)
logfile.write('\n\nCODE MARKUP THAT WONT BE LINKED (add entry to mapped_wiki_inline_code if one of these need to be linked. The tool get-api-items.py can be used to generate the list of the documented API items. \n')
for item in temp_set_of_codemarkup:
logfile.write('%s\n' % item)
# parser options
def main():
parser = optparse.OptionParser(version="%prog 0.1.1", usage="Usage: %prog [options] version")
parser.add_option("-c", "--clonewiki", action="store_true", default=False, dest="clonewiki", help="Clean and clone the latest wiki")
options, args = parser.parse_args()
print('Clone wiki: %s' % options.clonewiki)
if options.clonewiki:
CloneWiki()
# input = raw_input('CHECK ALL files were cloned! (look for "error: unable to create file" )\n')
ConvertFilesToRst()
FixupConvertedRstFiles()
print('See LOG for details: %s ' % logfilename)
if __name__ == '__main__':
sys.exit(main())
Computing file changes ...