Revision c1802c4afc6f5eace5197ea90fc3eb73a320ed03 authored by george-kuanli-peng on 16 October 2023, 10:15:42 UTC, committed by GitHub on 16 October 2023, 10:15:42 UTC
Fix the method name from `as_dict()` to `to_dict()` in SDK docs
1 parent 8b0130f
process_sdk_docs.py
#!/usr/bin/env python3
# Copyright (C) 2022 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
import argparse
import os
import os.path as osp
import re
import shutil
import sys
import textwrap
from glob import iglob
from typing import Callable, List
from inflection import underscore
class Processor:
_reference_files: List[str]
def __init__(self, *, input_dir: str, site_root: str) -> None:
self._input_dir = input_dir
self._site_root = site_root
self._content_dir = osp.join(self._site_root, "content")
self._sdk_reference_dir = osp.join(
self._content_dir, "en/docs/api_sdk/sdk/reference"
)
self._templates_dir = osp.join(self._site_root, "templates")
@staticmethod
def _copy_files(src_dir: str, glob_pattern: str, dst_dir: str) -> List[str]:
copied_files = []
for src_path in iglob(osp.join(src_dir, glob_pattern), recursive=True):
src_filename = osp.relpath(src_path, src_dir)
dst_path = osp.join(dst_dir, src_filename)
# assume dst dir exists
shutil.copy(src_path, dst_path, follow_symlinks=True)
copied_files.append(dst_path)
return copied_files
def _copy_pages(self):
self._reference_files = self._copy_files(
self._input_dir, "*/**/*.md", self._sdk_reference_dir
)
def _add_page_headers(self):
"""
Adds headers required by hugo to docs pages
"""
HEADER_SEPARATOR = "---"
for p in self._reference_files:
with open(p) as f:
contents = f.read()
assert not contents.startswith(HEADER_SEPARATOR), p
lines = contents.splitlines()
assert lines[0].startswith("#")
classname = lines[0][1:].strip()
header = textwrap.dedent(
"""\
%(header_separator)s
title: '%(classname)s class reference'
linkTitle: '%(classname)s'
weight: 10
description: ''
%(header_separator)s
"""
% {"header_separator": HEADER_SEPARATOR, "classname": classname}
)
contents = header + "\n".join(lines[1:])
with open(p, "w") as f:
f.write(contents)
def _move_api_summary(self):
"""
Moves API summary section from README to apis/_index
"""
SUMMARY_REPLACE_TOKEN = "{{REPLACEME:apis_summary}}" # nosec
with open(osp.join(self._input_dir, "api_summary.md")) as f:
apis_summary = f.read()
apis_index_filename = osp.join(
osp.relpath(self._sdk_reference_dir, self._content_dir), "apis/_index.md"
)
apis_index_path = osp.join(
self._templates_dir, apis_index_filename + ".template"
)
with open(apis_index_path) as f:
contents = f.read()
contents = contents.replace(SUMMARY_REPLACE_TOKEN, apis_summary)
with open(osp.join(self._content_dir, apis_index_filename), "w") as f:
f.write(contents)
def _fix_page_links_and_references(self):
"""
Replaces reference page links from full lowercase (which is generated by hugo from the
orignal camelcase and creates broken links) ('authapi') to the minus-case ('auth-api'),
which is more readable and works.
Adds an extra parent directory part to links ('../') as hugo requires, even for neighbor
files.
"""
mapping = {}
for src_path in self._reference_files:
src_filename = osp.relpath(src_path, self._sdk_reference_dir)
dst_filename = underscore(src_filename).replace("_", "-")
dst_path = osp.join(self._sdk_reference_dir, dst_filename)
os.rename(src_path, dst_path)
mapping[src_filename] = dst_filename
self._reference_files = [
osp.join(self._sdk_reference_dir, p) for p in mapping.values()
]
for p in iglob(self._sdk_reference_dir + "/**/*.md", recursive=True):
with open(p) as f:
contents = f.read()
for src_filename, dst_filename in mapping.items():
src_dir, src_filename = osp.split(osp.splitext(src_filename)[0])
dst_filename = osp.basename(osp.splitext(dst_filename)[0])
contents = re.sub(
rf"(\[.*?\]\()((?:\.\./)?(?:{src_dir}/)?){src_filename}((?:#[^\)]*?)?\))",
rf"\1../\2{dst_filename}\3",
contents,
)
with open(p, "w") as f:
f.write(contents)
def _process_non_code_blocks(
self, text: str, handlers: List[Callable[[str], str]]
) -> str:
"""
Allows to process Markdown documents with passed callbacks. Callbacks are only
executed outside code blocks.
"""
used_quotes = ""
block_start_pos = 0
inside_code_block = False
while block_start_pos < len(text):
pattern = re.compile(used_quotes or "```|`")
next_code_block_quote = pattern.search(text, pos=block_start_pos)
if next_code_block_quote is not None:
if not used_quotes:
inside_code_block = False
block_end_pos = next_code_block_quote.start(0)
used_quotes = next_code_block_quote.group(0)
else:
inside_code_block = True
block_end_pos = next_code_block_quote.end(0)
used_quotes = None
else:
block_end_pos = len(text)
if not inside_code_block:
block = text[block_start_pos:block_end_pos]
for handler in handlers:
block = handler(block)
text = text[:block_start_pos] + block + text[block_end_pos:]
block_end_pos = block_start_pos + len(block) + len(used_quotes)
block_start_pos = block_end_pos
return text
def _escape_free_square_brackets(self, text: str) -> str:
return re.sub(r"\[([^\[\]]*?)\]([^\(])", r"\[\1\]\2", text)
def _add_angle_brackets_to_free_links(self, text: str) -> str:
# Adapted from https://stackoverflow.com/a/31952097
URL_REGEX = (
# Scheme (HTTP, HTTPS):
r"(?:https?:\/\/)"
r"(?:"
# www:
r"(?:www\.)?"
# Host and domain (including ccSLD):
r"(?:(?:[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]\.)+)"
# TLD:
r"(?:[a-zA-Z]{2,6})"
# IP Address:
r"|(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
r")"
# Port:
r"(?::\d{1,5})?"
# Query path:
r"(?:(?:\/\S+)*|\/)"
)
text = re.sub(
r"(\A|[\.\s])(" + URL_REGEX + r")([\.\s]|\Z)",
r"\1<\2>\3",
text,
flags=re.MULTILINE,
)
return text
def _fix_parsing_problems(self):
"""
Adds angle brackets to freestanding links, as the linter requires. Such links can appear
from the generated model and api descriptions.
Adds escapes to freestanding square brackets to make parsing correct.
"""
for p in iglob(self._sdk_reference_dir + "/**/*.md", recursive=True):
with open(p) as f:
contents = f.read()
contents = self._process_non_code_blocks(
contents,
[
self._add_angle_brackets_to_free_links,
self._escape_free_square_brackets,
],
)
with open(p, "w") as f:
f.write(contents)
def run(self):
assert osp.isdir(self._input_dir), self._input_dir
assert osp.isdir(self._site_root), self._site_root
assert osp.isdir(self._sdk_reference_dir), self._sdk_reference_dir
assert osp.isdir(self._templates_dir), self._templates_dir
self._copy_pages()
self._move_api_summary()
self._add_page_headers()
self._fix_page_links_and_references()
self._fix_parsing_problems()
def parse_args(args=None):
parser = argparse.ArgumentParser()
parser.add_argument(
"--input-dir",
type=osp.abspath,
default="cvat-sdk/docs/",
help="Path to the cvat-sdk/docs/ directory",
)
parser.add_argument(
"--site-root",
type=osp.abspath,
default="site/",
)
return parser.parse_args(args)
def main(args=None):
args = parse_args(args)
processor = Processor(input_dir=args.input_dir, site_root=args.site_root)
processor.run()
return 0
if __name__ == "__main__":
sys.exit(main())
Computing file changes ...