Revision 6e32868127bf0c51e49efabf8b68ad544cae9a3c authored by Maria Khrustaleva on 01 December 2023, 12:05:17 UTC, committed by GitHub on 01 December 2023, 12:05:17 UTC
This PR resolves 2 issues related to rq jobs: 1. In some scenarios it is possible to reach a situation when `X2` rq job depends on `X1` -> running `X1` job is moved to FailedJobRegistry and not deleted -> user creates one more `X1` job that is enqueued after `X2` execution. It was possible due to the second issue when a user tried to export annotations for task 1, then for task 2, and after restarting the worker container user tried again to export annotations for task 1. (cyclic dependence) 2. Looks like in rq implementation rq jobs that depend on `X` job will never be enqueued when `X` is moved to FailedJobRegistry due to AbandonedJobError. I've submitted the [issue](https://github.com/rq/rq/issues/2006) to the rq repository.
1 parent 81e88e1
process_sdk_docs.py
#!/usr/bin/env python3
# Copyright (C) 2022 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
import argparse
import os
import os.path as osp
import re
import shutil
import sys
import textwrap
from glob import iglob
from typing import Callable, List
from inflection import underscore
class Processor:
_reference_files: List[str]
def __init__(self, *, input_dir: str, site_root: str) -> None:
self._input_dir = input_dir
self._site_root = site_root
self._content_dir = osp.join(self._site_root, "content")
self._sdk_reference_dir = osp.join(
self._content_dir, "en/docs/api_sdk/sdk/reference"
)
self._templates_dir = osp.join(self._site_root, "templates")
@staticmethod
def _copy_files(src_dir: str, glob_pattern: str, dst_dir: str) -> List[str]:
copied_files = []
for src_path in iglob(osp.join(src_dir, glob_pattern), recursive=True):
src_filename = osp.relpath(src_path, src_dir)
dst_path = osp.join(dst_dir, src_filename)
# assume dst dir exists
shutil.copy(src_path, dst_path, follow_symlinks=True)
copied_files.append(dst_path)
return copied_files
def _copy_pages(self):
self._reference_files = self._copy_files(
self._input_dir, "*/**/*.md", self._sdk_reference_dir
)
def _add_page_headers(self):
"""
Adds headers required by hugo to docs pages
"""
HEADER_SEPARATOR = "---"
for p in self._reference_files:
with open(p) as f:
contents = f.read()
assert not contents.startswith(HEADER_SEPARATOR), p
lines = contents.splitlines()
assert lines[0].startswith("#")
classname = lines[0][1:].strip()
header = textwrap.dedent(
"""\
%(header_separator)s
title: '%(classname)s class reference'
linkTitle: '%(classname)s'
weight: 10
description: ''
%(header_separator)s
"""
% {"header_separator": HEADER_SEPARATOR, "classname": classname}
)
contents = header + "\n".join(lines[1:])
with open(p, "w") as f:
f.write(contents)
def _move_api_summary(self):
"""
Moves API summary section from README to apis/_index
"""
SUMMARY_REPLACE_TOKEN = "{{REPLACEME:apis_summary}}" # nosec
with open(osp.join(self._input_dir, "api_summary.md")) as f:
apis_summary = f.read()
apis_index_filename = osp.join(
osp.relpath(self._sdk_reference_dir, self._content_dir), "apis/_index.md"
)
apis_index_path = osp.join(
self._templates_dir, apis_index_filename + ".template"
)
with open(apis_index_path) as f:
contents = f.read()
contents = contents.replace(SUMMARY_REPLACE_TOKEN, apis_summary)
with open(osp.join(self._content_dir, apis_index_filename), "w") as f:
f.write(contents)
def _fix_page_links_and_references(self):
"""
Replaces reference page links from full lowercase (which is generated by hugo from the
orignal camelcase and creates broken links) ('authapi') to the minus-case ('auth-api'),
which is more readable and works.
Adds an extra parent directory part to links ('../') as hugo requires, even for neighbor
files.
"""
mapping = {}
for src_path in self._reference_files:
src_filename = osp.relpath(src_path, self._sdk_reference_dir)
dst_filename = underscore(src_filename).replace("_", "-")
dst_path = osp.join(self._sdk_reference_dir, dst_filename)
os.rename(src_path, dst_path)
mapping[src_filename] = dst_filename
self._reference_files = [
osp.join(self._sdk_reference_dir, p) for p in mapping.values()
]
for p in iglob(self._sdk_reference_dir + "/**/*.md", recursive=True):
with open(p) as f:
contents = f.read()
for src_filename, dst_filename in mapping.items():
src_dir, src_filename = osp.split(osp.splitext(src_filename)[0])
dst_filename = osp.basename(osp.splitext(dst_filename)[0])
contents = re.sub(
rf"(\[.*?\]\()((?:\.\./)?(?:{src_dir}/)?){src_filename}((?:#[^\)]*?)?\))",
rf"\1../\2{dst_filename}\3",
contents,
)
with open(p, "w") as f:
f.write(contents)
def _process_non_code_blocks(
self, text: str, handlers: List[Callable[[str], str]]
) -> str:
"""
Allows to process Markdown documents with passed callbacks. Callbacks are only
executed outside code blocks.
"""
used_quotes = ""
block_start_pos = 0
inside_code_block = False
while block_start_pos < len(text):
pattern = re.compile(used_quotes or "```|`")
next_code_block_quote = pattern.search(text, pos=block_start_pos)
if next_code_block_quote is not None:
if not used_quotes:
inside_code_block = False
block_end_pos = next_code_block_quote.start(0)
used_quotes = next_code_block_quote.group(0)
else:
inside_code_block = True
block_end_pos = next_code_block_quote.end(0)
used_quotes = None
else:
block_end_pos = len(text)
if not inside_code_block:
block = text[block_start_pos:block_end_pos]
for handler in handlers:
block = handler(block)
text = text[:block_start_pos] + block + text[block_end_pos:]
block_end_pos = block_start_pos + len(block) + len(used_quotes)
block_start_pos = block_end_pos
return text
def _escape_free_square_brackets(self, text: str) -> str:
return re.sub(r"\[([^\[\]]*?)\]([^\(])", r"\[\1\]\2", text)
def _add_angle_brackets_to_free_links(self, text: str) -> str:
# Adapted from https://stackoverflow.com/a/31952097
URL_REGEX = (
# Scheme (HTTP, HTTPS):
r"(?:https?:\/\/)"
r"(?:"
# www:
r"(?:www\.)?"
# Host and domain (including ccSLD):
r"(?:(?:[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]\.)+)"
# TLD:
r"(?:[a-zA-Z]{2,6})"
# IP Address:
r"|(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
r")"
# Port:
r"(?::\d{1,5})?"
# Query path:
r"(?:(?:\/\S+)*|\/)"
)
text = re.sub(
r"(\A|[\.\s])(" + URL_REGEX + r")([\.\s]|\Z)",
r"\1<\2>\3",
text,
flags=re.MULTILINE,
)
return text
def _fix_parsing_problems(self):
"""
Adds angle brackets to freestanding links, as the linter requires. Such links can appear
from the generated model and api descriptions.
Adds escapes to freestanding square brackets to make parsing correct.
"""
for p in iglob(self._sdk_reference_dir + "/**/*.md", recursive=True):
with open(p) as f:
contents = f.read()
contents = self._process_non_code_blocks(
contents,
[
self._add_angle_brackets_to_free_links,
self._escape_free_square_brackets,
],
)
with open(p, "w") as f:
f.write(contents)
def run(self):
assert osp.isdir(self._input_dir), self._input_dir
assert osp.isdir(self._site_root), self._site_root
assert osp.isdir(self._sdk_reference_dir), self._sdk_reference_dir
assert osp.isdir(self._templates_dir), self._templates_dir
self._copy_pages()
self._move_api_summary()
self._add_page_headers()
self._fix_page_links_and_references()
self._fix_parsing_problems()
def parse_args(args=None):
parser = argparse.ArgumentParser()
parser.add_argument(
"--input-dir",
type=osp.abspath,
default="cvat-sdk/docs/",
help="Path to the cvat-sdk/docs/ directory",
)
parser.add_argument(
"--site-root",
type=osp.abspath,
default="site/",
)
return parser.parse_args(args)
def main(args=None):
args = parse_args(args)
processor = Processor(input_dir=args.input_dir, site_root=args.site_root)
processor.run()
return 0
if __name__ == "__main__":
sys.exit(main())
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...