back to Reference (Gold) summary
Reference (Gold): sphinx
Pytest Summary for test tests
| status | count |
|---|---|
| passed | 2187 |
| skipped | 22 |
| xfailed | 2 |
| total | 2211 |
| collected | 2211 |
Failed pytests:
test_command_line.py::test_build_main_parse_arguments_filenames_last
test_command_line.py::test_build_main_parse_arguments_filenames_last
@pytest.mark.xfail(reason='sphinx-build does not yet support filenames after options')
def test_build_main_parse_arguments_filenames_last() -> None:
args = [
*POSITIONAL_DIRS,
*OPTS,
*POSITIONAL_FILENAMES,
]
> assert parse_arguments(args) == EXPECTED_BUILD_MAIN
tests/test_command_line.py:126:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/test_command_line.py:86: in parse_arguments
parsed = vars(get_parser().parse_args(args))
/usr/lib/python3.10/argparse.py:1848: in parse_args
self.error(msg % ' '.join(argv))
/usr/lib/python3.10/argparse.py:2606: in error
self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = ArgumentParser(prog='pytest', usage='%(prog)s [OPTIONS] SOURCEDIR OUTPUTDIR [FILENAMES...]', description=i"\nGenerate ...ng individual filenames.\n", formatter_class=, conflict_handler='error', add_help=True)
status = 2
message = 'pytest: error: unrecognized arguments: filename1 filename2\n'
def exit(self, status=0, message=None):
if message:
self._print_message(message, _sys.stderr)
> _sys.exit(status)
E SystemExit: 2
/usr/lib/python3.10/argparse.py:2593: SystemExit
test_command_line.py::test_make_mode_parse_arguments_filenames_last
test_command_line.py::test_make_mode_parse_arguments_filenames_last
monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7ff2a4766140>
@pytest.mark.xfail(reason='sphinx-build does not yet support filenames after options')
def test_make_mode_parse_arguments_filenames_last(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(make_mode, 'build_main', parse_arguments)
args = [
*BUILDER_MAKE_MODE,
*POSITIONAL_DIRS,
*OPTS,
*POSITIONAL_FILENAMES,
]
> assert run_make_mode(args) == EXPECTED_MAKE_MODE
tests/test_command_line.py:199:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sphinx/cmd/make_mode.py:206: in run_make_mode
return make.run_generic_build(builder_name)
sphinx/cmd/make_mode.py:192: in run_generic_build
return build_main(args + self.opts)
tests/test_command_line.py:86: in parse_arguments
parsed = vars(get_parser().parse_args(args))
/usr/lib/python3.10/argparse.py:1848: in parse_args
self.error(msg % ' '.join(argv))
/usr/lib/python3.10/argparse.py:2606: in error
self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = ArgumentParser(prog='pytest', usage='%(prog)s [OPTIONS] SOURCEDIR OUTPUTDIR [FILENAMES...]', description=i"\nGenerate ...ng individual filenames.\n", formatter_class=, conflict_handler='error', add_help=True)
status = 2
message = 'pytest: error: unrecognized arguments: filename1 filename2\n'
def exit(self, status=0, message=None):
if message:
self._print_message(message, _sys.stderr)
> _sys.exit(status)
E SystemExit: 2
/usr/lib/python3.10/argparse.py:2593: SystemExit
Patch diff
diff --git a/sphinx/_cli/util/colour.py b/sphinx/_cli/util/colour.py
index 4ae735270..a89d04ec5 100644
--- a/sphinx/_cli/util/colour.py
+++ b/sphinx/_cli/util/colour.py
@@ -1,26 +1,89 @@
"""Format coloured console output."""
+
from __future__ import annotations
+
import os
import sys
-from collections.abc import Callable
+from collections.abc import Callable # NoQA: TCH003
+
if sys.platform == 'win32':
import colorama
+
+
_COLOURING_DISABLED = True
-def terminal_supports_colour() ->bool:
+def terminal_supports_colour() -> bool:
"""Return True if coloured terminal output is supported."""
- pass
+ if 'NO_COLOUR' in os.environ or 'NO_COLOR' in os.environ:
+ return False
+ if 'FORCE_COLOUR' in os.environ or 'FORCE_COLOR' in os.environ:
+ return True
+
+ try:
+ if not sys.stdout.isatty():
+ return False
+ except (AttributeError, ValueError):
+ # Handle cases where .isatty() is not defined, or where e.g.
+ # "ValueError: I/O operation on closed file" is raised
+ return False
+
+ # Do not colour output if on a dumb terminal
+ return os.environ.get('TERM', 'unknown').lower() not in {'dumb', 'unknown'}
+
+
+def disable_colour() -> None:
+ global _COLOURING_DISABLED
+ _COLOURING_DISABLED = True
+ if sys.platform == 'win32':
+ colorama.deinit()
+
+def enable_colour() -> None:
+ global _COLOURING_DISABLED
+ _COLOURING_DISABLED = False
+ if sys.platform == 'win32':
+ colorama.init()
+
+def colourise(colour_name: str, text: str, /) -> str:
+ if _COLOURING_DISABLED:
+ return text
+ return globals()[colour_name](text)
+
+
+def _create_colour_func(escape_code: str, /) -> Callable[[str], str]:
+ def inner(text: str) -> str:
+ if _COLOURING_DISABLED:
+ return text
+ return f'\x1b[{escape_code}m{text}\x1b[39;49;00m'
+ return inner
+
+
+# Wrap escape sequence with ``\1`` and ``\2`` to let readline know
+# that the colour escape codes are non-printable characters
+# [ https://tiswww.case.edu/php/chet/readline/readline.html ]
+#
+# Note: This does not work well in Windows
+# (see https://github.com/sphinx-doc/sphinx/pull/5059)
if sys.platform == 'win32':
_create_input_mode_colour_func = _create_colour_func
+else:
+ def _create_input_mode_colour_func(escape_code: str, /) -> Callable[[str], str]:
+ def inner(text: str) -> str:
+ if _COLOURING_DISABLED:
+ return text
+ return f'\x01\x1b[{escape_code}m\x02{text}\x01\x1b[39;49;00m\x02'
+ return inner
+
+
reset = _create_colour_func('39;49;00')
bold = _create_colour_func('01')
faint = _create_colour_func('02')
standout = _create_colour_func('03')
underline = _create_colour_func('04')
blink = _create_colour_func('05')
+
black = _create_colour_func('30')
darkred = _create_colour_func('31')
darkgreen = _create_colour_func('32')
@@ -29,6 +92,7 @@ darkblue = _create_colour_func('34')
purple = _create_colour_func('35')
turquoise = _create_colour_func('36')
lightgray = _create_colour_func('37')
+
darkgray = _create_colour_func('90')
red = _create_colour_func('91')
green = _create_colour_func('92')
diff --git a/sphinx/_cli/util/errors.py b/sphinx/_cli/util/errors.py
index c3e0cc08b..dac0fb83c 100644
--- a/sphinx/_cli/util/errors.py
+++ b/sphinx/_cli/util/errors.py
@@ -1,19 +1,165 @@
from __future__ import annotations
+
import re
import sys
import tempfile
from typing import TYPE_CHECKING, TextIO
+
from sphinx.errors import SphinxParallelError
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
+
_ANSI_COLOUR_CODES: re.Pattern[str] = re.compile('\x1b.*?m')
-def terminal_safe(s: str, /) ->str:
+def terminal_safe(s: str, /) -> str:
"""Safely encode a string for printing to the terminal."""
- pass
+ return s.encode('ascii', 'backslashreplace').decode('ascii')
+
+
+def strip_colors(s: str, /) -> str:
+ return _ANSI_COLOUR_CODES.sub('', s).strip()
+
+
+def error_info(messages: str, extensions: str, traceback: str) -> str:
+ import platform
+
+ import docutils
+ import jinja2
+ import pygments
+
+ import sphinx
+ return f"""\
+Versions
+========
-def save_traceback(app: (Sphinx | None), exc: BaseException) ->str:
+* Platform: {sys.platform}; ({platform.platform()})
+* Python version: {platform.python_version()} ({platform.python_implementation()})
+* Sphinx version: {sphinx.__display_version__}
+* Docutils version: {docutils.__version__}
+* Jinja2 version: {jinja2.__version__}
+* Pygments version: {pygments.__version__}
+
+Last Messages
+=============
+
+{messages}
+
+Loaded Extensions
+=================
+
+{extensions}
+
+Traceback
+=========
+
+{traceback}
+"""
+
+
+def save_traceback(app: Sphinx | None, exc: BaseException) -> str:
"""Save the given exception's traceback in a temporary file."""
- pass
+ if isinstance(exc, SphinxParallelError):
+ exc_format = '(Error in parallel process)\n' + exc.traceback
+ else:
+ import traceback
+
+ exc_format = traceback.format_exc()
+
+ last_msgs = exts_list = ''
+ if app is not None:
+ extensions = app.extensions.values()
+ last_msgs = '\n'.join(f'* {strip_colors(s)}' for s in app.messagelog)
+ exts_list = '\n'.join(f'* {ext.name} ({ext.version})' for ext in extensions
+ if ext.version != 'builtin')
+
+ with tempfile.NamedTemporaryFile(suffix='.log', prefix='sphinx-err-', delete=False) as f:
+ f.write(error_info(last_msgs, exts_list, exc_format).encode('utf-8'))
+
+ return f.name
+
+
+def handle_exception(
+ exception: BaseException,
+ /,
+ *,
+ stderr: TextIO = sys.stderr,
+ use_pdb: bool = False,
+ print_traceback: bool = False,
+ app: Sphinx | None = None,
+) -> None:
+ from bdb import BdbQuit
+ from traceback import TracebackException, print_exc
+
+ from docutils.utils import SystemMessage
+
+ from sphinx._cli.util.colour import red
+ from sphinx.errors import SphinxError
+ from sphinx.locale import __
+
+ if isinstance(exception, BdbQuit):
+ return
+
+ def print_err(*values: str) -> None:
+ print(*values, file=stderr)
+
+ def print_red(*values: str) -> None:
+ print_err(*map(red, values))
+
+ print_err()
+ if print_traceback or use_pdb:
+ print_exc(file=stderr)
+ print_err()
+
+ if use_pdb:
+ from pdb import post_mortem
+
+ print_red(__('Exception occurred, starting debugger:'))
+ post_mortem()
+ return
+
+ if isinstance(exception, KeyboardInterrupt):
+ print_err(__('Interrupted!'))
+ return
+
+ if isinstance(exception, SystemMessage):
+ print_red(__('reStructuredText markup error:'))
+ print_err(str(exception))
+ return
+
+ if isinstance(exception, SphinxError):
+ print_red(f'{exception.category}:')
+ print_err(str(exception))
+ return
+
+ if isinstance(exception, UnicodeError):
+ print_red(__('Encoding error:'))
+ print_err(str(exception))
+ return
+
+ if isinstance(exception, RecursionError):
+ print_red(__('Recursion error:'))
+ print_err(str(exception))
+ print_err()
+ print_err(__('This can happen with very large or deeply nested source '
+ 'files. You can carefully increase the default Python '
+ 'recursion limit of 1000 in conf.py with e.g.:'))
+ print_err('\n import sys\n sys.setrecursionlimit(1_500)\n')
+ return
+
+ # format an exception with traceback, but only the last frame.
+ te = TracebackException.from_exception(exception, limit=-1)
+ formatted_tb = te.stack.format()[-1] + ''.join(te.format_exception_only()).rstrip()
+
+ print_red(__('Exception occurred:'))
+ print_err(formatted_tb)
+ traceback_info_path = save_traceback(app, exception)
+ print_err(__('The full traceback has been saved in:'))
+ print_err(traceback_info_path)
+ print_err()
+ print_err(__('To report this error to the developers, please open an issue '
+ 'at <https://github.com/sphinx-doc/sphinx/issues/>. Thanks!'))
+ print_err(__('Please also report this if it was a user error, so '
+ 'that a better error message can be provided next time.'))
diff --git a/sphinx/addnodes.py b/sphinx/addnodes.py
index a277c8408..2a318aea7 100644
--- a/sphinx/addnodes.py
+++ b/sphinx/addnodes.py
@@ -1,10 +1,16 @@
"""Document tree nodes that Sphinx defines on top of those in Docutils."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any
+
from docutils import nodes
+
if TYPE_CHECKING:
from collections.abc import Sequence
+
from docutils.nodes import Element
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
@@ -19,6 +25,10 @@ class document(nodes.document):
in your extensions. It will be removed without deprecation period.
"""
+ def set_id(self, node: Element, msgnode: Element | None = None,
+ suggested_prefix: str = '') -> str:
+ return super().set_id(node, msgnode, suggested_prefix)
+
class translatable(nodes.Node):
"""Node which supports translation.
@@ -34,44 +44,84 @@ class translatable(nodes.Node):
Because they are used at final step; extraction.
"""
- def preserve_original_messages(self) ->None:
+ def preserve_original_messages(self) -> None:
"""Preserve original translatable messages."""
- pass
+ raise NotImplementedError
- def apply_translated_message(self, original_message: str,
- translated_message: str) ->None:
+ def apply_translated_message(self, original_message: str, translated_message: str) -> None:
"""Apply translated message."""
- pass
+ raise NotImplementedError
- def extract_original_messages(self) ->Sequence[str]:
+ def extract_original_messages(self) -> Sequence[str]:
"""Extract translation messages.
:returns: list of extracted messages or messages generator
"""
- pass
+ raise NotImplementedError
class not_smartquotable:
"""A node which does not support smart-quotes."""
+
support_smartquotes = False
class toctree(nodes.General, nodes.Element, translatable):
"""Node for inserting a "TOC tree"."""
+ def preserve_original_messages(self) -> None:
+ # toctree entries
+ rawentries: list[str] = self.setdefault('rawentries', [])
+ for title, _docname in self['entries']:
+ if title:
+ rawentries.append(title)
+
+ # :caption: option
+ if self.get('caption'):
+ self['rawcaption'] = self['caption']
+
+ def apply_translated_message(self, original_message: str, translated_message: str) -> None:
+ # toctree entries
+ for i, (title, docname) in enumerate(self['entries']):
+ if title == original_message:
+ self['entries'][i] = (translated_message, docname)
+
+ # :caption: option
+ if self.get('rawcaption') == original_message:
+ self['caption'] = translated_message
+
+ def extract_original_messages(self) -> list[str]:
+ messages: list[str] = []
+
+ # toctree entries
+ messages.extend(self.get('rawentries', []))
+
+ # :caption: option
+ if 'rawcaption' in self:
+ messages.append(self['rawcaption'])
+ return messages
+
+
+#############################################################
+# Domain-specific object descriptions (class, function etc.)
+#############################################################
class _desc_classes_injector(nodes.Element, not_smartquotable):
"""Helper base class for injecting a fixed list of classes.
Use as the first base class.
"""
+
classes: list[str] = []
- def __init__(self, *args: Any, **kwargs: Any) ->None:
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self['classes'].extend(self.classes)
+# Top-level nodes
+#################
+
class desc(nodes.Admonition, nodes.Element):
"""Node for a list of object signatures and a common description of them.
@@ -84,9 +134,11 @@ class desc(nodes.Admonition, nodes.Element):
- The name of the object type in the domain, e.g., ``function``.
"""
+ # TODO: can we introduce a constructor
+ # that forces the specification of the domain and objtyp?
-class desc_signature(_desc_classes_injector, nodes.Part, nodes.Inline,
- nodes.TextElement):
+
+class desc_signature(_desc_classes_injector, nodes.Part, nodes.Inline, nodes.TextElement):
"""Node for a single object signature.
As default the signature is a single-line signature.
@@ -95,8 +147,17 @@ class desc_signature(_desc_classes_injector, nodes.Part, nodes.Inline,
This node always has the classes ``sig``, ``sig-object``, and the domain it belongs to.
"""
+
+ # Note: the domain name is being added through a post-transform DescSigAddDomainAsClass
classes = ['sig', 'sig-object']
+ @property
+ def child_text_separator(self) -> str: # type: ignore[override]
+ if self.get('is_multiline'):
+ return ' '
+ else:
+ return super().child_text_separator
+
class desc_signature_line(nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for a line in a multi-line object signature.
@@ -105,6 +166,7 @@ class desc_signature_line(nodes.Part, nodes.Inline, nodes.FixedTextElement):
with ``is_multiline`` set to ``True``.
Set ``add_permalink = True`` for the line that should get the permalink.
"""
+
sphinx_line_type = ''
@@ -123,15 +185,20 @@ class desc_inline(_desc_classes_injector, nodes.Inline, nodes.TextElement):
This node always has the classes ``sig``, ``sig-inline``,
and the name of the domain it belongs to.
"""
+
classes = ['sig', 'sig-inline']
- def __init__(self, domain: str, *args: Any, **kwargs: Any) ->None:
+ def __init__(self, domain: str, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs, domain=domain)
self['classes'].append(domain)
-class desc_name(_desc_classes_injector, nodes.Part, nodes.Inline, nodes.
- FixedTextElement):
+# Nodes for high-level structure in signatures
+##############################################
+
+# nodes to use within a desc_signature or desc_signature_line
+
+class desc_name(_desc_classes_injector, nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for the main object name.
For example, in the declaration of a Python class ``MyModule.MyClass``,
@@ -139,11 +206,11 @@ class desc_name(_desc_classes_injector, nodes.Part, nodes.Inline, nodes.
This node always has the class ``sig-name``.
"""
- classes = ['sig-name', 'descname']
+
+ classes = ['sig-name', 'descname'] # 'descname' is for backwards compatibility
-class desc_addname(_desc_classes_injector, nodes.Part, nodes.Inline, nodes.
- FixedTextElement):
+class desc_addname(_desc_classes_injector, nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for additional name parts for an object.
For example, in the declaration of a Python class ``MyModule.MyClass``,
@@ -151,9 +218,12 @@ class desc_addname(_desc_classes_injector, nodes.Part, nodes.Inline, nodes.
This node always has the class ``sig-prename``.
"""
+
+ # 'descclassname' is for backwards compatibility
classes = ['sig-prename', 'descclassname']
+# compatibility alias
desc_classname = desc_addname
@@ -164,6 +234,9 @@ class desc_type(nodes.Part, nodes.Inline, nodes.FixedTextElement):
class desc_returns(desc_type):
"""Node for a "returns" annotation (a la -> in Python)."""
+ def astext(self) -> str:
+ return ' -> ' + super().astext()
+
class desc_parameterlist(nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for a general parameter list.
@@ -172,19 +245,26 @@ class desc_parameterlist(nodes.Part, nodes.Inline, nodes.FixedTextElement):
Set ``multi_line_parameter_list = True`` to describe a multi-line parameter list.
In that case each parameter will then be written on its own, indented line.
"""
+
child_text_separator = ', '
+ def astext(self) -> str:
+ return f'({super().astext()})'
-class desc_type_parameter_list(nodes.Part, nodes.Inline, nodes.FixedTextElement
- ):
+
+class desc_type_parameter_list(nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for a general type parameter list.
As default the type parameters list is written in line with the rest of the signature.
Set ``multi_line_parameter_list = True`` to describe a multi-line type parameters list.
In that case each type parameter will then be written on its own, indented line.
"""
+
child_text_separator = ', '
+ def astext(self) -> str:
+ return f'[{super().astext()}]'
+
class desc_parameter(nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for a single parameter."""
@@ -196,80 +276,115 @@ class desc_type_parameter(nodes.Part, nodes.Inline, nodes.FixedTextElement):
class desc_optional(nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for marking optional parts of the parameter list."""
+
child_text_separator = ', '
+ def astext(self) -> str:
+ return '[' + super().astext() + ']'
+
class desc_annotation(nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for signature annotations (not Python 3-style annotations)."""
+# Leaf nodes for markup of text fragments
+#########################################
+
+#: A set of classes inheriting :class:`desc_sig_element`. Each node class
+#: is expected to be handled by the builder's translator class if the latter
+#: does not inherit from SphinxTranslator.
+#:
+#: This set can be extended manually by third-party extensions or
+#: by subclassing :class:`desc_sig_element` and using the class
+#: keyword argument `_sig_element=True`.
SIG_ELEMENTS: set[type[desc_sig_element]] = set()
+# Signature text elements, generally translated to node.inline
+# in SigElementFallbackTransform.
+# When adding a new one, add it to SIG_ELEMENTS via the class
+# keyword argument `_sig_element=True` (e.g., see `desc_sig_space`).
+
class desc_sig_element(nodes.inline, _desc_classes_injector):
"""Common parent class of nodes for inline text of a signature."""
+
classes: list[str] = []
- def __init__(self, rawsource: str='', text: str='', *children: Element,
- **attributes: Any) ->None:
+ def __init__(self, rawsource: str = '', text: str = '',
+ *children: Element, **attributes: Any) -> None:
super().__init__(rawsource, text, *children, **attributes)
self['classes'].extend(self.classes)
- def __init_subclass__(cls, *, _sig_element: bool=False, **kwargs: Any
- ) ->None:
+ def __init_subclass__(cls, *, _sig_element: bool = False, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
if _sig_element:
+ # add the class to the SIG_ELEMENTS set if asked
SIG_ELEMENTS.add(cls)
-class desc_sig_space(desc_sig_element, _sig_element=(True)):
+# to not reinvent the wheel, the classes in the following desc_sig classes
+# are based on those used in Pygments
+
+class desc_sig_space(desc_sig_element, _sig_element=True):
"""Node for a space in a signature."""
- classes = ['w']
- def __init__(self, rawsource: str='', text: str=' ', *children: Element,
- **attributes: Any) ->None:
+ classes = ["w"]
+
+ def __init__(self, rawsource: str = '', text: str = ' ',
+ *children: Element, **attributes: Any) -> None:
super().__init__(rawsource, text, *children, **attributes)
-class desc_sig_name(desc_sig_element, _sig_element=(True)):
+class desc_sig_name(desc_sig_element, _sig_element=True):
"""Node for an identifier in a signature."""
- classes = ['n']
+
+ classes = ["n"]
-class desc_sig_operator(desc_sig_element, _sig_element=(True)):
+class desc_sig_operator(desc_sig_element, _sig_element=True):
"""Node for an operator in a signature."""
- classes = ['o']
+
+ classes = ["o"]
-class desc_sig_punctuation(desc_sig_element, _sig_element=(True)):
+class desc_sig_punctuation(desc_sig_element, _sig_element=True):
"""Node for punctuation in a signature."""
- classes = ['p']
+ classes = ["p"]
-class desc_sig_keyword(desc_sig_element, _sig_element=(True)):
+
+class desc_sig_keyword(desc_sig_element, _sig_element=True):
"""Node for a general keyword in a signature."""
- classes = ['k']
+
+ classes = ["k"]
-class desc_sig_keyword_type(desc_sig_element, _sig_element=(True)):
+class desc_sig_keyword_type(desc_sig_element, _sig_element=True):
"""Node for a keyword which is a built-in type in a signature."""
- classes = ['kt']
+ classes = ["kt"]
-class desc_sig_literal_number(desc_sig_element, _sig_element=(True)):
+
+class desc_sig_literal_number(desc_sig_element, _sig_element=True):
"""Node for a numeric literal in a signature."""
- classes = ['m']
+
+ classes = ["m"]
-class desc_sig_literal_string(desc_sig_element, _sig_element=(True)):
+class desc_sig_literal_string(desc_sig_element, _sig_element=True):
"""Node for a string literal in a signature."""
- classes = ['s']
+ classes = ["s"]
-class desc_sig_literal_char(desc_sig_element, _sig_element=(True)):
+
+class desc_sig_literal_char(desc_sig_element, _sig_element=True):
"""Node for a character literal in a signature."""
- classes = ['sc']
+ classes = ["sc"]
+
+
+###############################################################
+# new admonition-like constructs
class versionmodified(nodes.Admonition, nodes.TextElement):
"""Node for version change entries.
@@ -294,6 +409,8 @@ class production(nodes.Part, nodes.Inline, nodes.FixedTextElement):
"""Node for a single grammar production rule."""
+# other directive-level nodes
+
class index(nodes.Invisible, nodes.Inline, nodes.TextElement):
"""Node for index entries.
@@ -339,6 +456,8 @@ class only(nodes.Element):
"""Node for "only" directives (conditional inclusion based on tags)."""
+# meta-information nodes
+
class start_of_file(nodes.Element):
"""Node to mark start of a new file, used in the LaTeX builder only."""
@@ -353,6 +472,8 @@ class tabular_col_spec(nodes.Element):
"""Node for specifying tabular columns, used for LaTeX output."""
+# inline nodes
+
class pending_xref(nodes.Inline, nodes.Element):
"""Node for cross-references that cannot be resolved without complete
information about all documents.
@@ -360,6 +481,7 @@ class pending_xref(nodes.Inline, nodes.Element):
These nodes are resolved before writing output, in
BuildEnvironment.resolve_references.
"""
+
child_text_separator = ''
@@ -432,3 +554,55 @@ class literal_strong(nodes.strong, not_smartquotable):
class manpage(nodes.Inline, nodes.FixedTextElement):
"""Node for references to manpages."""
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_node(toctree)
+
+ app.add_node(desc)
+ app.add_node(desc_signature)
+ app.add_node(desc_signature_line)
+ app.add_node(desc_content)
+ app.add_node(desc_inline)
+
+ app.add_node(desc_name)
+ app.add_node(desc_addname)
+ app.add_node(desc_type)
+ app.add_node(desc_returns)
+ app.add_node(desc_parameterlist)
+ app.add_node(desc_type_parameter_list)
+ app.add_node(desc_parameter)
+ app.add_node(desc_type_parameter)
+ app.add_node(desc_optional)
+ app.add_node(desc_annotation)
+
+ for n in SIG_ELEMENTS:
+ app.add_node(n)
+
+ app.add_node(versionmodified)
+ app.add_node(seealso)
+ app.add_node(productionlist)
+ app.add_node(production)
+ app.add_node(index)
+ app.add_node(centered)
+ app.add_node(acks)
+ app.add_node(hlist)
+ app.add_node(hlistcol)
+ app.add_node(compact_paragraph)
+ app.add_node(glossary)
+ app.add_node(only)
+ app.add_node(start_of_file)
+ app.add_node(highlightlang)
+ app.add_node(tabular_col_spec)
+ app.add_node(pending_xref)
+ app.add_node(number_reference)
+ app.add_node(download_reference)
+ app.add_node(literal_emphasis)
+ app.add_node(literal_strong)
+ app.add_node(manpage)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/application.py b/sphinx/application.py
index c0830a77e..ea1f79ba7 100644
--- a/sphinx/application.py
+++ b/sphinx/application.py
@@ -2,7 +2,9 @@
Gracefully adapted from the TextPress system by Armin.
"""
+
from __future__ import annotations
+
import contextlib
import os
import pickle
@@ -11,7 +13,9 @@ from collections import deque
from io import StringIO
from os import path
from typing import TYPE_CHECKING, overload
+
from docutils.parsers.rst import Directive, roles
+
import sphinx
from sphinx import locale, package_dir
from sphinx.config import ENUM, Config, _ConfigRebuild
@@ -31,15 +35,18 @@ from sphinx.util.i18n import CatalogRepository
from sphinx.util.logging import prefixed_warnings
from sphinx.util.osutil import ensuredir, relpath
from sphinx.util.tags import Tags
+
if TYPE_CHECKING:
from collections.abc import Callable, Collection, Iterable, Sequence, Set
from pathlib import Path
from typing import IO, Any, Final, Literal
+
from docutils import nodes
from docutils.nodes import Element, Node
from docutils.parsers import Parser
from docutils.transforms import Transform
from pygments.lexer import Lexer
+
from sphinx import addnodes
from sphinx.builders import Builder
from sphinx.domains import Domain, Index
@@ -51,38 +58,74 @@ if TYPE_CHECKING:
from sphinx.search import SearchLanguage
from sphinx.theming import Theme
from sphinx.util.typing import RoleFunction, TitleGetter
-builtin_extensions: tuple[str, ...] = ('sphinx.addnodes',
- 'sphinx.builders.changes', 'sphinx.builders.epub3',
- 'sphinx.builders.dirhtml', 'sphinx.builders.dummy',
- 'sphinx.builders.gettext', 'sphinx.builders.html',
- 'sphinx.builders.latex', 'sphinx.builders.linkcheck',
- 'sphinx.builders.manpage', 'sphinx.builders.singlehtml',
- 'sphinx.builders.texinfo', 'sphinx.builders.text',
- 'sphinx.builders.xml', 'sphinx.config', 'sphinx.domains.c',
- 'sphinx.domains.changeset', 'sphinx.domains.citation',
- 'sphinx.domains.cpp', 'sphinx.domains.index',
- 'sphinx.domains.javascript', 'sphinx.domains.math',
- 'sphinx.domains.python', 'sphinx.domains.rst', 'sphinx.domains.std',
- 'sphinx.directives', 'sphinx.directives.code',
- 'sphinx.directives.other', 'sphinx.directives.patches',
- 'sphinx.extension', 'sphinx.parsers', 'sphinx.registry', 'sphinx.roles',
- 'sphinx.transforms', 'sphinx.transforms.compact_bullet_list',
- 'sphinx.transforms.i18n', 'sphinx.transforms.references',
+
+
+builtin_extensions: tuple[str, ...] = (
+ 'sphinx.addnodes',
+ 'sphinx.builders.changes',
+ 'sphinx.builders.epub3',
+ 'sphinx.builders.dirhtml',
+ 'sphinx.builders.dummy',
+ 'sphinx.builders.gettext',
+ 'sphinx.builders.html',
+ 'sphinx.builders.latex',
+ 'sphinx.builders.linkcheck',
+ 'sphinx.builders.manpage',
+ 'sphinx.builders.singlehtml',
+ 'sphinx.builders.texinfo',
+ 'sphinx.builders.text',
+ 'sphinx.builders.xml',
+ 'sphinx.config',
+ 'sphinx.domains.c',
+ 'sphinx.domains.changeset',
+ 'sphinx.domains.citation',
+ 'sphinx.domains.cpp',
+ 'sphinx.domains.index',
+ 'sphinx.domains.javascript',
+ 'sphinx.domains.math',
+ 'sphinx.domains.python',
+ 'sphinx.domains.rst',
+ 'sphinx.domains.std',
+ 'sphinx.directives',
+ 'sphinx.directives.code',
+ 'sphinx.directives.other',
+ 'sphinx.directives.patches',
+ 'sphinx.extension',
+ 'sphinx.parsers',
+ 'sphinx.registry',
+ 'sphinx.roles',
+ 'sphinx.transforms',
+ 'sphinx.transforms.compact_bullet_list',
+ 'sphinx.transforms.i18n',
+ 'sphinx.transforms.references',
'sphinx.transforms.post_transforms',
'sphinx.transforms.post_transforms.code',
- 'sphinx.transforms.post_transforms.images', 'sphinx.versioning',
+ 'sphinx.transforms.post_transforms.images',
+ 'sphinx.versioning',
+ # collectors should be loaded by specific order
'sphinx.environment.collectors.dependencies',
'sphinx.environment.collectors.asset',
'sphinx.environment.collectors.metadata',
'sphinx.environment.collectors.title',
- 'sphinx.environment.collectors.toctree')
-_first_party_extensions = ('sphinxcontrib.applehelp',
- 'sphinxcontrib.devhelp', 'sphinxcontrib.htmlhelp',
- 'sphinxcontrib.serializinghtml', 'sphinxcontrib.qthelp')
-_first_party_themes = 'alabaster',
+ 'sphinx.environment.collectors.toctree',
+)
+_first_party_extensions = (
+ # 1st party extensions
+ 'sphinxcontrib.applehelp',
+ 'sphinxcontrib.devhelp',
+ 'sphinxcontrib.htmlhelp',
+ 'sphinxcontrib.serializinghtml',
+ 'sphinxcontrib.qthelp',
+)
+_first_party_themes = (
+ # Alabaster is loaded automatically to be used as the default theme
+ 'alabaster',
+)
builtin_extensions += _first_party_themes
builtin_extensions += _first_party_extensions
+
ENV_PICKLE_FILENAME = 'environment.pickle'
+
logger = logging.getLogger(__name__)
@@ -94,17 +137,18 @@ class Sphinx:
:ivar doctreedir: Directory for storing pickled doctrees.
:ivar outdir: Directory for storing build documents.
"""
+
warningiserror: Final = False
_warncount: int
- def __init__(self, srcdir: (str | os.PathLike[str]), confdir: (str | os
- .PathLike[str] | None), outdir: (str | os.PathLike[str]),
- doctreedir: (str | os.PathLike[str]), buildername: str,
- confoverrides: (dict | None)=None, status: (IO[str] | None)=sys.
- stdout, warning: (IO[str] | None)=sys.stderr, freshenv: bool=False,
- warningiserror: bool=False, tags: Sequence[str]=(), verbosity: int=
- 0, parallel: int=0, keep_going: bool=False, pdb: bool=False,
- exception_on_warning: bool=False) ->None:
+ def __init__(self, srcdir: str | os.PathLike[str], confdir: str | os.PathLike[str] | None,
+ outdir: str | os.PathLike[str], doctreedir: str | os.PathLike[str],
+ buildername: str, confoverrides: dict | None = None,
+ status: IO[str] | None = sys.stdout, warning: IO[str] | None = sys.stderr,
+ freshenv: bool = False, warningiserror: bool = False,
+ tags: Sequence[str] = (),
+ verbosity: int = 0, parallel: int = 0, keep_going: bool = False,
+ pdb: bool = False, exception_on_warning: bool = False) -> None:
"""Initialize the Sphinx application.
:param srcdir: The path to the source directory.
@@ -132,103 +176,268 @@ class Sphinx:
self._fresh_env_used: bool | None = None
self.extensions: dict[str, Extension] = {}
self.registry = SphinxComponentRegistry()
+
+ # validate provided directories
self.srcdir = _StrPath(srcdir).resolve()
self.outdir = _StrPath(outdir).resolve()
self.doctreedir = _StrPath(doctreedir).resolve()
+
if not path.isdir(self.srcdir):
raise ApplicationError(__('Cannot find source directory (%s)') %
- self.srcdir)
+ self.srcdir)
+
if path.exists(self.outdir) and not path.isdir(self.outdir):
- raise ApplicationError(__(
- 'Output directory (%s) is not a directory') % self.outdir)
+ raise ApplicationError(__('Output directory (%s) is not a directory') %
+ self.outdir)
+
if self.srcdir == self.outdir:
- raise ApplicationError(__(
- 'Source directory and destination directory cannot be identical'
- ))
+ raise ApplicationError(__('Source directory and destination '
+ 'directory cannot be identical'))
+
self.parallel = parallel
+
if status is None:
self._status: IO[str] = StringIO()
self.quiet: bool = True
else:
self._status = status
self.quiet = False
+
if warning is None:
self._warning: IO[str] = StringIO()
else:
self._warning = warning
self._warncount = 0
- self.keep_going = bool(warningiserror)
+ self.keep_going = bool(warningiserror) # Unused
self._fail_on_warnings = bool(warningiserror)
self.pdb = pdb
self._exception_on_warning = exception_on_warning
logging.setup(self, self._status, self._warning)
+
self.events = EventManager(self)
+
+ # keep last few messages for traceback
+ # This will be filled by sphinx.util.logging.LastMessagesWriter
self.messagelog: deque = deque(maxlen=10)
+
+ # say hello to the world
logger.info(bold(__('Running Sphinx v%s')), sphinx.__display_version__)
+
+ # status code for command-line application
self.statuscode = 0
+
+ # read config
self.tags = Tags(tags)
if confdir is None:
+ # set confdir to srcdir if -C given (!= no confdir); a few pieces
+ # of code expect a confdir to be set
self.confdir = self.srcdir
self.config = Config({}, confoverrides or {})
else:
self.confdir = _StrPath(confdir).resolve()
- self.config = Config.read(self.confdir, confoverrides or {},
- self.tags)
+ self.config = Config.read(self.confdir, confoverrides or {}, self.tags)
+
+ # set up translation infrastructure
self._init_i18n()
- if (self.config.needs_sphinx and self.config.needs_sphinx > sphinx.
- __display_version__):
- raise VersionRequirementError(__(
- 'This project needs at least Sphinx v%s and therefore cannot be built with this version.'
- ) % self.config.needs_sphinx)
+
+ # check the Sphinx version if requested
+ if self.config.needs_sphinx and self.config.needs_sphinx > sphinx.__display_version__:
+ raise VersionRequirementError(
+ __('This project needs at least Sphinx v%s and therefore cannot '
+ 'be built with this version.') % self.config.needs_sphinx)
+
+ # load all built-in extension modules, first-party extension modules,
+ # and first-party themes
for extension in builtin_extensions:
self.setup_extension(extension)
+
+ # load all user-given extension modules
for extension in self.config.extensions:
self.setup_extension(extension)
+
+ # preload builder module (before init config values)
self.preload_builder(buildername)
+
if not path.isdir(outdir):
with progress_message(__('making output directory')):
ensuredir(outdir)
+
+ # the config file itself can be an extension
if self.config.setup:
- prefix = __('while setting up extension %s:') % 'conf.py'
+ prefix = __('while setting up extension %s:') % "conf.py"
with prefixed_warnings(prefix):
if callable(self.config.setup):
self.config.setup(self)
else:
- raise ConfigError(__(
- "'setup' as currently defined in conf.py isn't a Python callable. Please modify its definition to make it a callable function. This is needed for conf.py to behave as a Sphinx extension."
- ))
+ raise ConfigError(
+ __("'setup' as currently defined in conf.py isn't a Python callable. "
+ "Please modify its definition to make it a callable function. "
+ "This is needed for conf.py to behave as a Sphinx extension."),
+ )
+
+ # Report any warnings for overrides.
self.config._report_override_warnings()
self.events.emit('config-inited', self.config)
+
+ # create the project
self.project = Project(self.srcdir, self.config.source_suffix)
+
+ # set up the build environment
self.env = self._init_env(freshenv)
+
+ # create the builder
self.builder = self.create_builder(buildername)
+
+ # build environment post-initialisation, after creating the builder
self._post_init_env()
+
+ # set up the builder
self._init_builder()
@property
- def fresh_env_used(self) ->(bool | None):
+ def fresh_env_used(self) -> bool | None:
"""True/False as to whether a new environment was created for this build,
or None if the environment has not been initialised yet.
"""
- pass
+ return self._fresh_env_used
- def _init_i18n(self) ->None:
+ def _init_i18n(self) -> None:
"""Load translated strings from the configured localedirs if enabled in
the configuration.
"""
- pass
+ logger.info(bold(__('loading translations [%s]... ')), self.config.language,
+ nonl=True)
+
+ # compile mo files if sphinx.po file in user locale directories are updated
+ repo = CatalogRepository(self.srcdir, self.config.locale_dirs,
+ self.config.language, self.config.source_encoding)
+ for catalog in repo.catalogs:
+ if catalog.domain == 'sphinx' and catalog.is_outdated():
+ catalog.write_mo(self.config.language,
+ self.config.gettext_allow_fuzzy_translations)
+
+ locale_dirs: list[str | None] = list(repo.locale_dirs)
+ locale_dirs += [None]
+ locale_dirs += [path.join(package_dir, 'locale')]
+
+ self.translator, has_translation = locale.init(locale_dirs, self.config.language)
+ if has_translation or self.config.language == 'en':
+ logger.info(__('done'))
+ else:
+ logger.info(__('not available for built-in messages'))
- def setup_extension(self, extname: str) ->None:
+ def _init_env(self, freshenv: bool) -> BuildEnvironment:
+ filename = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
+ if freshenv or not os.path.exists(filename):
+ return self._create_fresh_env()
+ else:
+ return self._load_existing_env(filename)
+
+ def _create_fresh_env(self) -> BuildEnvironment:
+ env = BuildEnvironment(self)
+ self._fresh_env_used = True
+ return env
+
+ @progress_message(__('loading pickled environment'))
+ def _load_existing_env(self, filename: str) -> BuildEnvironment:
+ try:
+ with open(filename, 'rb') as f:
+ env = pickle.load(f)
+ env.setup(self)
+ self._fresh_env_used = False
+ except Exception as err:
+ logger.info(__('failed: %s'), err)
+ env = self._create_fresh_env()
+ return env
+
+ def _post_init_env(self) -> None:
+ if self._fresh_env_used:
+ self.env.find_files(self.config, self.builder)
+
+ def preload_builder(self, name: str) -> None:
+ self.registry.preload_builder(self, name)
+
+ def create_builder(self, name: str) -> Builder:
+ if name is None:
+ logger.info(__('No builder selected, using default: html'))
+ name = 'html'
+
+ return self.registry.create_builder(self, name, self.env)
+
+ def _init_builder(self) -> None:
+ self.builder.init()
+ self.events.emit('builder-inited')
+
+ # ---- main "build" method -------------------------------------------------
+
+ def build(self, force_all: bool = False, filenames: list[str] | None = None) -> None:
+ self.phase = BuildPhase.READING
+ try:
+ if force_all:
+ self.builder.build_all()
+ elif filenames:
+ self.builder.build_specific(filenames)
+ else:
+ self.builder.build_update()
+
+ self.events.emit('build-finished', None)
+ except Exception as err:
+ # delete the saved env to force a fresh build next time
+ envfile = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
+ if path.isfile(envfile):
+ os.unlink(envfile)
+ self.events.emit('build-finished', err)
+ raise
+
+ if self._warncount == 0:
+ if self.statuscode != 0:
+ logger.info(bold(__('build finished with problems.')))
+ else:
+ logger.info(bold(__('build succeeded.')))
+ elif self._warncount == 1:
+ if self._fail_on_warnings:
+ self.statuscode = 1
+ msg = __('build finished with problems, 1 warning '
+ '(with warnings treated as errors).')
+ elif self.statuscode != 0:
+ msg = __('build finished with problems, 1 warning.')
+ else:
+ msg = __('build succeeded, 1 warning.')
+ logger.info(bold(msg))
+ else:
+ if self._fail_on_warnings:
+ self.statuscode = 1
+ msg = __('build finished with problems, %s warnings '
+ '(with warnings treated as errors).')
+ elif self.statuscode != 0:
+ msg = __('build finished with problems, %s warnings.')
+ else:
+ msg = __('build succeeded, %s warnings.')
+ logger.info(bold(msg), self._warncount)
+
+ if self.statuscode == 0 and self.builder.epilog:
+ logger.info('')
+ logger.info(self.builder.epilog, {
+ 'outdir': relpath(self.outdir),
+ 'project': self.config.project,
+ })
+
+ self.builder.cleanup()
+
+ # ---- general extensibility interface -------------------------------------
+
+ def setup_extension(self, extname: str) -> None:
"""Import and setup a Sphinx extension module.
Load the extension given by the module *name*. Use this if your
extension needs the features provided by another extension. No-op if
called twice.
"""
- pass
+ logger.debug('[app] setting up extension: %r', extname)
+ self.registry.load_extension(self, extname)
@staticmethod
- def require_sphinx(version: (tuple[int, int] | str)) ->None:
+ def require_sphinx(version: tuple[int, int] | str) -> None:
"""Check the Sphinx version if requested.
Compare *version* with the version of the running Sphinx, and abort the
@@ -241,9 +450,339 @@ class Sphinx:
.. versionchanged:: 7.1
Type of *version* now allows ``(major, minor)`` form.
"""
- pass
-
- def connect(self, event: str, callback: Callable, priority: int=500) ->int:
+ if isinstance(version, tuple):
+ major, minor = version
+ else:
+ major, minor = map(int, version.split('.')[:2])
+ if (major, minor) > sphinx.version_info[:2]:
+ req = f'{major}.{minor}'
+ raise VersionRequirementError(req)
+
+ # ---- Core events -------------------------------------------------------
+
+ @overload
+ def connect(
+ self,
+ event: Literal['config-inited'],
+ callback: Callable[[Sphinx, Config], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['builder-inited'],
+ callback: Callable[[Sphinx], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['env-get-outdated'],
+ callback: Callable[
+ [Sphinx, BuildEnvironment, Set[str], Set[str], Set[str]], Sequence[str]
+ ],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['env-before-read-docs'],
+ callback: Callable[[Sphinx, BuildEnvironment, list[str]], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['env-purge-doc'],
+ callback: Callable[[Sphinx, BuildEnvironment, str], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['source-read'],
+ callback: Callable[[Sphinx, str, list[str]], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['include-read'],
+ callback: Callable[[Sphinx, Path, str, list[str]], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['doctree-read'],
+ callback: Callable[[Sphinx, nodes.document], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['env-merge-info'],
+ callback: Callable[
+ [Sphinx, BuildEnvironment, list[str], BuildEnvironment], None
+ ],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['env-updated'],
+ callback: Callable[[Sphinx, BuildEnvironment], str],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['env-get-updated'],
+ callback: Callable[[Sphinx, BuildEnvironment], Iterable[str]],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['env-check-consistency'],
+ callback: Callable[[Sphinx, BuildEnvironment], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['write-started'],
+ callback: Callable[[Sphinx, Builder], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['doctree-resolved'],
+ callback: Callable[[Sphinx, nodes.document, str], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['missing-reference'],
+ callback: Callable[
+ [Sphinx, BuildEnvironment, addnodes.pending_xref, nodes.TextElement],
+ nodes.reference | None,
+ ],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['warn-missing-reference'],
+ callback: Callable[[Sphinx, Domain, addnodes.pending_xref], bool | None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['build-finished'],
+ callback: Callable[[Sphinx, Exception | None], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ # ---- Events from builtin builders --------------------------------------
+
+ @overload
+ def connect(
+ self,
+ event: Literal['html-collect-pages'],
+ callback: Callable[[Sphinx], Iterable[tuple[str, dict[str, Any], str]]],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['html-page-context'],
+ callback: Callable[
+ [Sphinx, str, str, dict[str, Any], nodes.document], str | None
+ ],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['linkcheck-process-uri'],
+ callback: Callable[[Sphinx, str], str | None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ # ---- Events from builtin extensions-- ----------------------------------
+
+ @overload
+ def connect(
+ self,
+ event: Literal['object-description-transform'],
+ callback: Callable[[Sphinx, str, str, addnodes.desc_content], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ # ---- Events from first-party extensions --------------------------------
+
+ @overload
+ def connect(
+ self,
+ event: Literal['autodoc-process-docstring'],
+ callback: Callable[
+ [
+ Sphinx,
+ Literal['module', 'class', 'exception', 'function', 'method', 'attribute'],
+ str,
+ Any,
+ dict[str, bool],
+ Sequence[str],
+ ],
+ None,
+ ],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['autodoc-before-process-signature'],
+ callback: Callable[[Sphinx, Any, bool], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['autodoc-process-signature'],
+ callback: Callable[
+ [
+ Sphinx,
+ Literal['module', 'class', 'exception', 'function', 'method', 'attribute'],
+ str,
+ Any,
+ dict[str, bool],
+ str | None,
+ str | None,
+ ],
+ tuple[str | None, str | None] | None,
+ ],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['autodoc-process-bases'],
+ callback: Callable[[Sphinx, str, Any, dict[str, bool], list[str]], None],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['autodoc-skip-member'],
+ callback: Callable[
+ [
+ Sphinx,
+ Literal['module', 'class', 'exception', 'function', 'method', 'attribute'],
+ str,
+ Any,
+ bool,
+ dict[str, bool],
+ ],
+ bool,
+ ],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['todo-defined'],
+ callback: Callable[[Sphinx, todo_node], None],
+ priority: int = 500,
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['viewcode-find-source'],
+ callback: Callable[
+ [Sphinx, str],
+ tuple[str, dict[str, tuple[Literal['class', 'def', 'other'], int, int]]],
+ ],
+ priority: int = 500,
+ ) -> int:
+ ...
+
+ @overload
+ def connect(
+ self,
+ event: Literal['viewcode-follow-imported'],
+ callback: Callable[[Sphinx, str, str], str | None],
+ priority: int = 500,
+ ) -> int:
+ ...
+
+ # ---- Catch-all ---------------------------------------------------------
+
+ @overload
+ def connect(
+ self,
+ event: str,
+ callback: Callable[..., Any],
+ priority: int = 500
+ ) -> int:
+ ...
+
+ # event interface
+ def connect(self, event: str, callback: Callable, priority: int = 500) -> int:
"""Register *callback* to be called when *event* is emitted.
For details on available core events and the arguments of callback
@@ -259,17 +798,21 @@ class Sphinx:
Support *priority*
"""
- pass
+ listener_id = self.events.connect(event, callback, priority)
+ logger.debug('[app] connecting event %r (%d): %r [id=%s]',
+ event, priority, callback, listener_id)
+ return listener_id
- def disconnect(self, listener_id: int) ->None:
+ def disconnect(self, listener_id: int) -> None:
"""Unregister callback by *listener_id*.
:param listener_id: A listener_id that :meth:`connect` returns
"""
- pass
+ logger.debug('[app] disconnecting event: [id=%s]', listener_id)
+ self.events.disconnect(listener_id)
- def emit(self, event: str, *args: Any, allowed_exceptions: tuple[type[
- Exception], ...]=()) ->list:
+ def emit(self, event: str, *args: Any,
+ allowed_exceptions: tuple[type[Exception], ...] = ()) -> list:
"""Emit *event* and pass *arguments* to the callback functions.
Return the return values of all callbacks as a list. Do not emit core
@@ -283,10 +826,10 @@ class Sphinx:
Added *allowed_exceptions* to specify path-through exceptions
"""
- pass
+ return self.events.emit(event, *args, allowed_exceptions=allowed_exceptions)
- def emit_firstresult(self, event: str, *args: Any, allowed_exceptions:
- tuple[type[Exception], ...]=()) ->Any:
+ def emit_firstresult(self, event: str, *args: Any,
+ allowed_exceptions: tuple[type[Exception], ...] = ()) -> Any:
"""Emit *event* and pass *arguments* to the callback functions.
Return the result of the first callback that doesn't return ``None``.
@@ -300,9 +843,12 @@ class Sphinx:
Added *allowed_exceptions* to specify path-through exceptions
"""
- pass
+ return self.events.emit_firstresult(event, *args,
+ allowed_exceptions=allowed_exceptions)
- def add_builder(self, builder: type[Builder], override: bool=False) ->None:
+ # registering addon parts
+
+ def add_builder(self, builder: type[Builder], override: bool = False) -> None:
"""Register a new builder.
:param builder: A builder class
@@ -312,11 +858,13 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ self.registry.add_builder(builder, override=override)
- def add_config_value(self, name: str, default: Any, rebuild:
- _ConfigRebuild, types: (type | Collection[type] | ENUM)=(),
- description: str='') ->None:
+ def add_config_value(
+ self, name: str, default: Any, rebuild: _ConfigRebuild,
+ types: type | Collection[type] | ENUM = (),
+ description: str = '',
+ ) -> None:
"""Register a configuration value.
This is necessary for Sphinx to recognize new values and set default
@@ -353,19 +901,21 @@ class Sphinx:
.. versionadded:: 7.4
The *description* parameter.
"""
- pass
+ logger.debug('[app] adding config value: %r', (name, default, rebuild, types))
+ self.config.add(name, default, rebuild, types, description)
- def add_event(self, name: str) ->None:
+ def add_event(self, name: str) -> None:
"""Register an event called *name*.
This is needed to be able to emit it.
:param name: The name of the event
"""
- pass
+ logger.debug('[app] adding event: %r', name)
+ self.events.add(name)
- def set_translator(self, name: str, translator_class: type[nodes.
- NodeVisitor], override: bool=False) ->None:
+ def set_translator(self, name: str, translator_class: type[nodes.NodeVisitor],
+ override: bool = False) -> None:
"""Register or override a Docutils translator class.
This is used to register a custom output translator or to replace a
@@ -381,10 +931,10 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ self.registry.add_translator(name, translator_class, override=override)
- def add_node(self, node: type[Element], override: bool=False, **kwargs:
- tuple[Callable, Callable | None]) ->None:
+ def add_node(self, node: type[Element], override: bool = False,
+ **kwargs: tuple[Callable, Callable | None]) -> None:
"""Register a Docutils node class.
This is necessary for Docutils internals. It may also be used in the
@@ -419,11 +969,17 @@ class Sphinx:
.. versionchanged:: 0.5
Added the support for keyword arguments giving visit functions.
"""
- pass
+ logger.debug('[app] adding node: %r', (node, kwargs))
+ if not override and docutils.is_node_registered(node):
+ logger.warning(__('node class %r is already registered, '
+ 'its visitors will be overridden'),
+ node.__name__, type='app', subtype='add_node')
+ docutils.register_node(node)
+ self.registry.add_translation_handlers(node, **kwargs)
def add_enumerable_node(self, node: type[Element], figtype: str,
- title_getter: (TitleGetter | None)=None, override: bool=False, **
- kwargs: tuple[Callable, Callable]) ->None:
+ title_getter: TitleGetter | None = None, override: bool = False,
+ **kwargs: tuple[Callable, Callable]) -> None:
"""Register a Docutils node class as a numfig target.
Sphinx numbers the node automatically. And then the users can refer it
@@ -447,10 +1003,10 @@ class Sphinx:
.. versionadded:: 1.4
"""
- pass
+ self.registry.add_enumerable_node(node, figtype, title_getter, override=override)
+ self.add_node(node, override=override, **kwargs)
- def add_directive(self, name: str, cls: type[Directive], override: bool
- =False) ->None:
+ def add_directive(self, name: str, cls: type[Directive], override: bool = False) -> None:
"""Register a Docutils directive.
:param name: The name of the directive
@@ -492,9 +1048,14 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ logger.debug('[app] adding directive: %r', (name, cls))
+ if not override and docutils.is_directive_registered(name):
+ logger.warning(__('directive %r is already registered, it will be overridden'),
+ name, type='app', subtype='add_directive')
+
+ docutils.register_directive(name, cls)
- def add_role(self, name: str, role: Any, override: bool=False) ->None:
+ def add_role(self, name: str, role: Any, override: bool = False) -> None:
"""Register a Docutils role.
:param name: The name of role
@@ -509,10 +1070,16 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ logger.debug('[app] adding role: %r', (name, role))
+ if not override and docutils.is_role_registered(name):
+ logger.warning(__('role %r is already registered, it will be overridden'),
+ name, type='app', subtype='add_role')
+ docutils.register_role(name, role)
- def add_generic_role(self, name: str, nodeclass: type[Node], override:
- bool=False) ->None:
+ def add_generic_role(
+ self, name: str, nodeclass: type[Node], override: bool = False
+
+ ) -> None:
"""Register a generic Docutils role.
Register a Docutils role that does nothing but wrap its contents in the
@@ -526,9 +1093,16 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
-
- def add_domain(self, domain: type[Domain], override: bool=False) ->None:
+ # Don't use ``roles.register_generic_role`` because it uses
+ # ``register_canonical_role``.
+ logger.debug('[app] adding generic role: %r', (name, nodeclass))
+ if not override and docutils.is_role_registered(name):
+ logger.warning(__('role %r is already registered, it will be overridden'),
+ name, type='app', subtype='add_generic_role')
+ role = roles.GenericRole(name, nodeclass)
+ docutils.register_role(name, role)
+
+ def add_domain(self, domain: type[Domain], override: bool = False) -> None:
"""Register a domain.
:param domain: A domain class
@@ -540,10 +1114,10 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ self.registry.add_domain(domain, override=override)
- def add_directive_to_domain(self, domain: str, name: str, cls: type[
- Directive], override: bool=False) ->None:
+ def add_directive_to_domain(self, domain: str, name: str,
+ cls: type[Directive], override: bool = False) -> None:
"""Register a Docutils directive in a domain.
Like :meth:`add_directive`, but the directive is added to the domain
@@ -560,10 +1134,10 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ self.registry.add_directive_to_domain(domain, name, cls, override=override)
- def add_role_to_domain(self, domain: str, name: str, role: (
- RoleFunction | XRefRole), override: bool=False) ->None:
+ def add_role_to_domain(self, domain: str, name: str, role: RoleFunction | XRefRole,
+ override: bool = False) -> None:
"""Register a Docutils role in a domain.
Like :meth:`add_role`, but the role is added to the domain named
@@ -580,10 +1154,10 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ self.registry.add_role_to_domain(domain, name, role, override=override)
- def add_index_to_domain(self, domain: str, index: type[Index],
- _override: bool=False) ->None:
+ def add_index_to_domain(self, domain: str, index: type[Index], _override: bool = False,
+ ) -> None:
"""Register a custom index for a domain.
Add a custom *index* class to the domain named *domain*.
@@ -598,12 +1172,14 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
-
- def add_object_type(self, directivename: str, rolename: str,
- indextemplate: str='', parse_node: (Callable | None)=None,
- ref_nodeclass: (type[nodes.TextElement] | None)=None, objname: str=
- '', doc_field_types: Sequence=(), override: bool=False) ->None:
+ self.registry.add_index_to_domain(domain, index)
+
+ def add_object_type(self, directivename: str, rolename: str, indextemplate: str = '',
+ parse_node: Callable | None = None,
+ ref_nodeclass: type[nodes.TextElement] | None = None,
+ objname: str = '', doc_field_types: Sequence = (),
+ override: bool = False,
+ ) -> None:
"""Register a new object type.
This method is a very convenient way to add a new :term:`object` type
@@ -663,11 +1239,15 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
-
- def add_crossref_type(self, directivename: str, rolename: str,
- indextemplate: str='', ref_nodeclass: (type[nodes.TextElement] |
- None)=None, objname: str='', override: bool=False) ->None:
+ self.registry.add_object_type(directivename, rolename, indextemplate, parse_node,
+ ref_nodeclass, objname, doc_field_types,
+ override=override)
+
+ def add_crossref_type(
+ self, directivename: str, rolename: str, indextemplate: str = '',
+ ref_nodeclass: type[nodes.TextElement] | None = None, objname: str = '',
+ override: bool = False,
+ ) -> None:
"""Register a new crossref object type.
This method is very similar to :meth:`~Sphinx.add_object_type` except that the
@@ -702,9 +1282,11 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ self.registry.add_crossref_type(directivename, rolename,
+ indextemplate, ref_nodeclass, objname,
+ override=override)
- def add_transform(self, transform: type[Transform]) ->None:
+ def add_transform(self, transform: type[Transform]) -> None:
"""Register a Docutils transform to be applied after parsing.
Add the standard docutils :class:`~docutils.transforms.Transform`
@@ -736,10 +1318,10 @@ class Sphinx:
refs: `Transform Priority Range Categories`__
__ https://docutils.sourceforge.io/docs/ref/transforms.html#transform-priority-range-categories
- """
- pass
+ """ # NoQA: E501,RUF100 # Flake8 thinks the URL is too long, Ruff special cases URLs.
+ self.registry.add_transform(transform)
- def add_post_transform(self, transform: type[Transform]) ->None:
+ def add_post_transform(self, transform: type[Transform]) -> None:
"""Register a Docutils transform to be applied before writing.
Add the standard docutils :class:`~docutils.transforms.Transform`
@@ -748,10 +1330,10 @@ class Sphinx:
:param transform: A transform class
"""
- pass
+ self.registry.add_post_transform(transform)
- def add_js_file(self, filename: (str | None), priority: int=500,
- loading_method: (str | None)=None, **kwargs: Any) ->None:
+ def add_js_file(self, filename: str | None, priority: int = 500,
+ loading_method: str | None = None, **kwargs: Any) -> None:
"""Register a JavaScript file to include in the HTML output.
:param filename: The name of a JavaScript file that the default HTML
@@ -809,10 +1391,18 @@ class Sphinx:
Take loading_method argument. Allow to change the loading method of the
JavaScript file.
"""
- pass
-
- def add_css_file(self, filename: str, priority: int=500, **kwargs: Any
- ) ->None:
+ if loading_method == 'async':
+ kwargs['async'] = 'async'
+ elif loading_method == 'defer':
+ kwargs['defer'] = 'defer'
+
+ self.registry.add_js_file(filename, priority=priority, **kwargs)
+ with contextlib.suppress(AttributeError):
+ self.builder.add_js_file( # type: ignore[attr-defined]
+ filename, priority=priority, **kwargs,
+ )
+
+ def add_css_file(self, filename: str, priority: int = 500, **kwargs: Any) -> None:
"""Register a stylesheet to include in the HTML output.
:param filename: The name of a CSS file that the default HTML
@@ -869,32 +1459,37 @@ class Sphinx:
.. versionchanged:: 3.5
Take priority argument. Allow to add a CSS file to the specific page.
"""
- pass
+ logger.debug('[app] adding stylesheet: %r', filename)
+ self.registry.add_css_files(filename, priority=priority, **kwargs)
+ with contextlib.suppress(AttributeError):
+ self.builder.add_css_file( # type: ignore[attr-defined]
+ filename, priority=priority, **kwargs,
+ )
- def add_latex_package(self, packagename: str, options: (str | None)=
- None, after_hyperref: bool=False) ->None:
- """Register a package to include in the LaTeX source code.
+ def add_latex_package(self, packagename: str, options: str | None = None,
+ after_hyperref: bool = False) -> None:
+ r"""Register a package to include in the LaTeX source code.
Add *packagename* to the list of packages that LaTeX source code will
- include. If you provide *options*, it will be taken to the `\\usepackage`
+ include. If you provide *options*, it will be taken to the `\usepackage`
declaration. If you set *after_hyperref* truthy, the package will be
loaded after ``hyperref`` package.
.. code-block:: python
app.add_latex_package('mypackage')
- # => \\usepackage{mypackage}
+ # => \usepackage{mypackage}
app.add_latex_package('mypackage', 'foo,bar')
- # => \\usepackage[foo,bar]{mypackage}
+ # => \usepackage[foo,bar]{mypackage}
.. versionadded:: 1.3
.. versionadded:: 3.1
*after_hyperref* option.
"""
- pass
+ self.registry.add_latex_package(packagename, options, after_hyperref)
- def add_lexer(self, alias: str, lexer: type[Lexer]) ->None:
+ def add_lexer(self, alias: str, lexer: type[Lexer]) -> None:
"""Register a new lexer for source code.
Use *lexer* to highlight code blocks with the given language *alias*.
@@ -905,10 +1500,10 @@ class Sphinx:
.. versionchanged:: 4.0
Removed support for lexer instances as an argument.
"""
- pass
+ logger.debug('[app] adding lexer: %r', (alias, lexer))
+ lexer_classes[alias] = lexer
- def add_autodocumenter(self, cls: type[Documenter], override: bool=False
- ) ->None:
+ def add_autodocumenter(self, cls: type[Documenter], override: bool = False) -> None:
"""Register a new documenter class for the autodoc extension.
Add *cls* as a new documenter class for the :mod:`sphinx.ext.autodoc`
@@ -926,10 +1521,13 @@ class Sphinx:
.. versionchanged:: 2.2
Add *override* keyword.
"""
- pass
+ logger.debug('[app] adding autodocumenter: %r', cls)
+ from sphinx.ext.autodoc.directive import AutodocDirective
+ self.registry.add_documenter(cls.objtype, cls)
+ self.add_directive('auto' + cls.objtype, AutodocDirective, override=override)
- def add_autodoc_attrgetter(self, typ: type, getter: Callable[[Any, str,
- Any], Any]) ->None:
+ def add_autodoc_attrgetter(self, typ: type, getter: Callable[[Any, str, Any], Any],
+ ) -> None:
"""Register a new ``getattr``-like function for the autodoc extension.
Add *getter*, which must be a function with an interface compatible to
@@ -940,9 +1538,10 @@ class Sphinx:
.. versionadded:: 0.6
"""
- pass
+ logger.debug('[app] adding autodoc attrgetter: %r', (typ, getter))
+ self.registry.add_autodoc_attrgetter(typ, getter)
- def add_search_language(self, cls: type[SearchLanguage]) ->None:
+ def add_search_language(self, cls: type[SearchLanguage]) -> None:
"""Register a new language for the HTML search index.
Add *cls*, which must be a subclass of
@@ -953,10 +1552,11 @@ class Sphinx:
.. versionadded:: 1.1
"""
- pass
+ logger.debug('[app] adding search language: %r', cls)
+ from sphinx.search import languages
+ languages[cls.lang] = cls
- def add_source_suffix(self, suffix: str, filetype: str, override: bool=
- False) ->None:
+ def add_source_suffix(self, suffix: str, filetype: str, override: bool = False) -> None:
"""Register a suffix of source files.
Same as :confval:`source_suffix`. The users can override this
@@ -968,10 +1568,9 @@ class Sphinx:
.. versionadded:: 1.8
"""
- pass
+ self.registry.add_source_suffix(suffix, filetype, override=override)
- def add_source_parser(self, parser: type[Parser], override: bool=False
- ) ->None:
+ def add_source_parser(self, parser: type[Parser], override: bool = False) -> None:
"""Register a parser class.
:param override: If false, do not install it if another parser
@@ -985,18 +1584,19 @@ class Sphinx:
.. versionchanged:: 1.8
Add *override* keyword.
"""
- pass
+ self.registry.add_source_parser(parser, override=override)
- def add_env_collector(self, collector: type[EnvironmentCollector]) ->None:
+ def add_env_collector(self, collector: type[EnvironmentCollector]) -> None:
"""Register an environment collector class.
Refer to :ref:`collector-api`.
.. versionadded:: 1.6
"""
- pass
+ logger.debug('[app] adding environment collector: %r', collector)
+ collector().enable(self)
- def add_html_theme(self, name: str, theme_path: str) ->None:
+ def add_html_theme(self, name: str, theme_path: str) -> None:
"""Register a HTML Theme.
The *name* is a name of theme, and *theme_path* is a full path to the
@@ -1004,11 +1604,15 @@ class Sphinx:
.. versionadded:: 1.6
"""
- pass
-
- def add_html_math_renderer(self, name: str, inline_renderers: (tuple[
- Callable, Callable | None] | None)=None, block_renderers: (tuple[
- Callable, Callable | None] | None)=None) ->None:
+ logger.debug('[app] adding HTML theme: %r, %r', name, theme_path)
+ self.registry.add_html_theme(name, theme_path)
+
+ def add_html_math_renderer(
+ self,
+ name: str,
+ inline_renderers: tuple[Callable, Callable | None] | None = None,
+ block_renderers: tuple[Callable, Callable | None] | None = None,
+ ) -> None:
"""Register a math renderer for HTML.
The *name* is a name of math renderer. Both *inline_renderers* and
@@ -1020,9 +1624,9 @@ class Sphinx:
.. versionadded:: 1.8
"""
- pass
+ self.registry.add_html_math_renderer(name, inline_renderers, block_renderers)
- def add_message_catalog(self, catalog: str, locale_dir: str) ->None:
+ def add_message_catalog(self, catalog: str, locale_dir: str) -> None:
"""Register a message catalog.
:param catalog: The name of the catalog
@@ -1032,17 +1636,46 @@ class Sphinx:
.. versionadded:: 1.8
"""
- pass
+ locale.init([locale_dir], self.config.language, catalog)
+ locale.init_console(locale_dir, catalog)
- def is_parallel_allowed(self, typ: str) ->bool:
+ # ---- other methods -------------------------------------------------
+ def is_parallel_allowed(self, typ: str) -> bool:
"""Check whether parallel processing is allowed or not.
:param typ: A type of processing; ``'read'`` or ``'write'``.
"""
- pass
-
- def set_html_assets_policy(self, policy: Literal['always', 'per_page']
- ) ->None:
+ if typ == 'read':
+ attrname = 'parallel_read_safe'
+ message_not_declared = __("the %s extension does not declare if it "
+ "is safe for parallel reading, assuming "
+ "it isn't - please ask the extension author "
+ "to check and make it explicit")
+ message_not_safe = __("the %s extension is not safe for parallel reading")
+ elif typ == 'write':
+ attrname = 'parallel_write_safe'
+ message_not_declared = __("the %s extension does not declare if it "
+ "is safe for parallel writing, assuming "
+ "it isn't - please ask the extension author "
+ "to check and make it explicit")
+ message_not_safe = __("the %s extension is not safe for parallel writing")
+ else:
+ raise ValueError('parallel type %s is not supported' % typ)
+
+ for ext in self.extensions.values():
+ allowed = getattr(ext, attrname, None)
+ if allowed is None:
+ logger.warning(message_not_declared, ext.name)
+ logger.warning(__('doing serial %s'), typ)
+ return False
+ elif not allowed:
+ logger.warning(message_not_safe, ext.name)
+ logger.warning(__('doing serial %s'), typ)
+ return False
+
+ return True
+
+ def set_html_assets_policy(self, policy: Literal['always', 'per_page']) -> None:
"""Set the policy to include assets in HTML pages.
- always: include the assets in all the pages
@@ -1050,7 +1683,9 @@ class Sphinx:
.. versionadded: 4.1
"""
- pass
+ if policy not in ('always', 'per_page'):
+ raise ValueError('policy %s is not supported' % policy)
+ self.registry.html_assets_policy = policy
class TemplateBridge:
@@ -1059,8 +1694,12 @@ class TemplateBridge:
that renders templates given a template name and a context.
"""
- def init(self, builder: Builder, theme: (Theme | None)=None, dirs: (
- list[str] | None)=None) ->None:
+ def init(
+ self,
+ builder: Builder,
+ theme: Theme | None = None,
+ dirs: list[str] | None = None,
+ ) -> None:
"""Called by the builder to initialize the template system.
*builder* is the builder object; you'll probably want to look at the
@@ -1069,23 +1708,26 @@ class TemplateBridge:
*theme* is a :class:`sphinx.theming.Theme` object or None; in the latter
case, *dirs* can be list of fixed directories to look for templates.
"""
- pass
+ msg = 'must be implemented in subclasses'
+ raise NotImplementedError(msg)
- def newest_template_mtime(self) ->float:
+ def newest_template_mtime(self) -> float:
"""Called by the builder to determine if output files are outdated
because of template changes. Return the mtime of the newest template
file that was changed. The default implementation returns ``0``.
"""
- pass
+ return 0
- def render(self, template: str, context: dict) ->None:
+ def render(self, template: str, context: dict) -> None:
"""Called by the builder to render a template given as a filename with
a specified context (a Python dictionary).
"""
- pass
+ msg = 'must be implemented in subclasses'
+ raise NotImplementedError(msg)
- def render_string(self, template: str, context: dict) ->str:
+ def render_string(self, template: str, context: dict) -> str:
"""Called by the builder to render a template given as a string with a
specified context (a Python dictionary).
"""
- pass
+ msg = 'must be implemented in subclasses'
+ raise NotImplementedError(msg)
diff --git a/sphinx/builders/_epub_base.py b/sphinx/builders/_epub_base.py
index aa543c3e3..15c4bd80b 100644
--- a/sphinx/builders/_epub_base.py
+++ b/sphinx/builders/_epub_base.py
@@ -1,5 +1,7 @@
"""Base class of epub2/epub3 builders."""
+
from __future__ import annotations
+
import html
import os
import re
@@ -8,8 +10,10 @@ from os import path
from typing import TYPE_CHECKING, Any, NamedTuple
from urllib.parse import quote
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
+
from docutils import nodes
from docutils.utils import smartquotes
+
from sphinx import addnodes
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.builders.html._build_info import BuildInfo
@@ -18,27 +22,64 @@ from sphinx.util import logging
from sphinx.util.display import status_iterator
from sphinx.util.fileutil import copy_asset_file
from sphinx.util.osutil import copyfile, ensuredir, relpath
+
if TYPE_CHECKING:
from docutils.nodes import Element, Node
+
try:
from PIL import Image
PILLOW_AVAILABLE = True
except ImportError:
PILLOW_AVAILABLE = False
+
+
logger = logging.getLogger(__name__)
+
+
+# (Fragment) templates from which the metainfo files content.opf and
+# toc.ncx are created.
+# This template section also defines strings that are embedded in the html
+# output but that may be customized by (re-)setting module attributes,
+# e.g. from conf.py.
+
COVERPAGE_NAME = 'epub-cover.xhtml'
+
TOCTREE_TEMPLATE = 'toctree-l%d'
+
LINK_TARGET_TEMPLATE = ' [%(uri)s]'
+
FOOTNOTE_LABEL_TEMPLATE = '#%d'
+
FOOTNOTES_RUBRIC_NAME = 'Footnotes'
+
CSS_LINK_TARGET_CLASS = 'link-target'
-GUIDE_TITLES = {'toc': 'Table of Contents', 'cover': 'Cover'}
-MEDIA_TYPES = {'.xhtml': 'application/xhtml+xml', '.css': 'text/css',
- '.png': 'image/png', '.webp': 'image/webp', '.gif': 'image/gif', '.svg':
- 'image/svg+xml', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.otf':
- 'font/otf', '.ttf': 'font/ttf', '.woff': 'font/woff'}
-VECTOR_GRAPHICS_EXTENSIONS = '.svg',
-REFURI_RE = re.compile('([^#:]*#)(.*)')
+
+# XXX These strings should be localized according to epub_language
+GUIDE_TITLES = {
+ 'toc': 'Table of Contents',
+ 'cover': 'Cover',
+}
+
+MEDIA_TYPES = {
+ '.xhtml': 'application/xhtml+xml',
+ '.css': 'text/css',
+ '.png': 'image/png',
+ '.webp': 'image/webp',
+ '.gif': 'image/gif',
+ '.svg': 'image/svg+xml',
+ '.jpg': 'image/jpeg',
+ '.jpeg': 'image/jpeg',
+ '.otf': 'font/otf',
+ '.ttf': 'font/ttf',
+ '.woff': 'font/woff',
+}
+
+VECTOR_GRAPHICS_EXTENSIONS = ('.svg',)
+
+# Regular expression to match colons only in local fragment identifiers.
+# If the URI contains a colon before the #,
+# it is an external link that should not change.
+REFURI_RE = re.compile("([^#:]*#)(.*)")
class ManifestItem(NamedTuple):
@@ -66,9 +107,19 @@ class NavPoint(NamedTuple):
children: list[NavPoint]
+def sphinx_smarty_pants(t: str, language: str = 'en') -> str:
+ t = t.replace('"', '"')
+ t = smartquotes.educateDashesOldSchool(t) # type: ignore[no-untyped-call]
+ t = smartquotes.educateQuotes(t, language) # type: ignore[no-untyped-call]
+ t = t.replace('"', '"')
+ return t
+
+
ssp = sphinx_smarty_pants
+# The epub publisher
+
class EpubBuilder(StandaloneHTMLBuilder):
"""
Builder that outputs epub files.
@@ -77,16 +128,26 @@ class EpubBuilder(StandaloneHTMLBuilder):
META-INF/container.xml. Afterwards, all necessary files are zipped to an
epub file.
"""
+
+ # don't copy the reST source
copysource = False
supported_image_types = ['image/svg+xml', 'image/png', 'image/gif',
- 'image/jpeg']
+ 'image/jpeg']
supported_remote_images = False
+
+ # don't add links
add_permalinks = False
+ # don't use # as current path. ePub check reject it.
allow_sharp_as_current_path = False
+ # don't add sidebar etc.
embedded = True
+ # disable download role
download_support = False
+ # don't create links to original images from images
html_scaled_image_link = False
+ # don't generate search index or include search page
search = False
+
coverpage_name = COVERPAGE_NAME
toctree_template = TOCTREE_TEMPLATE
link_target_template = LINK_TARGET_TEMPLATE
@@ -94,134 +155,592 @@ class EpubBuilder(StandaloneHTMLBuilder):
guide_titles = GUIDE_TITLES
media_types = MEDIA_TYPES
refuri_re = REFURI_RE
- template_dir = ''
- doctype = ''
-
- def make_id(self, name: str) ->str:
+ template_dir = ""
+ doctype = ""
+
+ def init(self) -> None:
+ super().init()
+ # the output files for epub must be .html only
+ self.out_suffix = '.xhtml'
+ self.link_suffix = '.xhtml'
+ self.playorder = 0
+ self.tocid = 0
+ self.id_cache: dict[str, str] = {}
+ self.use_index = self.get_builder_config('use_index', 'epub')
+ self.refnodes: list[dict[str, Any]] = []
+
+ def create_build_info(self) -> BuildInfo:
+ return BuildInfo(self.config, self.tags, frozenset({'html', 'epub'}))
+
+ def get_theme_config(self) -> tuple[str, dict[str, str | int | bool]]:
+ return self.config.epub_theme, self.config.epub_theme_options
+
+ # generic support functions
+ def make_id(self, name: str) -> str:
+ # id_cache is intentionally mutable
"""Return a unique id for name."""
- pass
-
- def get_refnodes(self, doctree: Node, result: list[dict[str, Any]]) ->list[
- dict[str, Any]]:
+ id = self.id_cache.get(name)
+ if not id:
+ id = 'epub-%d' % self.env.new_serialno('epub')
+ self.id_cache[name] = id
+ return id
+
+ def get_refnodes(
+ self, doctree: Node, result: list[dict[str, Any]],
+ ) -> list[dict[str, Any]]:
"""Collect section titles, their depth in the toc and the refuri."""
- pass
-
- def get_toc(self) ->None:
+ # XXX: is there a better way than checking the attribute
+ # toctree-l[1-8] on the parent node?
+ if isinstance(doctree, nodes.reference) and doctree.get('refuri'):
+ refuri = doctree['refuri']
+ if refuri.startswith(('http://', 'https://', 'irc:', 'mailto:')):
+ return result
+ classes = doctree.parent.attributes['classes']
+ for level in range(8, 0, -1): # or range(1, 8)?
+ if (self.toctree_template % level) in classes:
+ result.append({
+ 'level': level,
+ 'refuri': html.escape(refuri),
+ 'text': ssp(html.escape(doctree.astext())),
+ })
+ break
+ elif isinstance(doctree, nodes.Element):
+ for elem in doctree:
+ result = self.get_refnodes(elem, result)
+ return result
+
+ def check_refnodes(self, nodes: list[dict[str, Any]]) -> None:
+ appeared: set[str] = set()
+ for node in nodes:
+ if node['refuri'] in appeared:
+ logger.warning(
+ __('duplicated ToC entry found: %s'),
+ node['refuri'],
+ type="epub",
+ subtype="duplicated_toc_entry",
+ )
+ else:
+ appeared.add(node['refuri'])
+
+ def get_toc(self) -> None:
"""Get the total table of contents, containing the root_doc
and pre and post files not managed by sphinx.
"""
- pass
-
- def toc_add_files(self, refnodes: list[dict[str, Any]]) ->None:
+ doctree = self.env.get_and_resolve_doctree(self.config.root_doc,
+ self, prune_toctrees=False,
+ includehidden=True)
+ self.refnodes = self.get_refnodes(doctree, [])
+ master_dir = path.dirname(self.config.root_doc)
+ if master_dir:
+ master_dir += '/' # XXX or os.sep?
+ for item in self.refnodes:
+ item['refuri'] = master_dir + item['refuri']
+ self.toc_add_files(self.refnodes)
+
+ def toc_add_files(self, refnodes: list[dict[str, Any]]) -> None:
"""Add the root_doc, pre and post files to a list of refnodes.
"""
- pass
-
- def fix_fragment(self, prefix: str, fragment: str) ->str:
+ refnodes.insert(0, {
+ 'level': 1,
+ 'refuri': html.escape(self.config.root_doc + self.out_suffix),
+ 'text': ssp(html.escape(
+ self.env.titles[self.config.root_doc].astext())),
+ })
+ for file, text in reversed(self.config.epub_pre_files):
+ refnodes.insert(0, {
+ 'level': 1,
+ 'refuri': html.escape(file),
+ 'text': ssp(html.escape(text)),
+ })
+ for file, text in self.config.epub_post_files:
+ refnodes.append({
+ 'level': 1,
+ 'refuri': html.escape(file),
+ 'text': ssp(html.escape(text)),
+ })
+
+ def fix_fragment(self, prefix: str, fragment: str) -> str:
"""Return a href/id attribute with colons replaced by hyphens."""
- pass
+ return prefix + fragment.replace(':', '-')
- def fix_ids(self, tree: nodes.document) ->None:
+ def fix_ids(self, tree: nodes.document) -> None:
"""Replace colons with hyphens in href and id attributes.
Some readers crash because they interpret the part as a
transport protocol specification.
"""
- pass
-
- def add_visible_links(self, tree: nodes.document, show_urls: str='inline'
- ) ->None:
+ def update_node_id(node: Element) -> None:
+ """Update IDs of given *node*."""
+ new_ids: list[str] = []
+ for node_id in node['ids']:
+ new_id = self.fix_fragment('', node_id)
+ if new_id not in new_ids:
+ new_ids.append(new_id)
+ node['ids'] = new_ids
+
+ for reference in tree.findall(nodes.reference):
+ if 'refuri' in reference:
+ m = self.refuri_re.match(reference['refuri'])
+ if m:
+ reference['refuri'] = self.fix_fragment(m.group(1), m.group(2))
+ if 'refid' in reference:
+ reference['refid'] = self.fix_fragment('', reference['refid'])
+
+ for target in tree.findall(nodes.target):
+ update_node_id(target)
+
+ next_node: Node = target.next_node(ascend=True)
+ if isinstance(next_node, nodes.Element):
+ update_node_id(next_node)
+
+ for desc_signature in tree.findall(addnodes.desc_signature):
+ update_node_id(desc_signature)
+
+ def add_visible_links(self, tree: nodes.document, show_urls: str = 'inline') -> None:
"""Add visible link targets for external links"""
- pass
- def write_doc(self, docname: str, doctree: nodes.document) ->None:
+ def make_footnote_ref(doc: nodes.document, label: str) -> nodes.footnote_reference:
+ """Create a footnote_reference node with children"""
+ footnote_ref = nodes.footnote_reference('[#]_')
+ footnote_ref.append(nodes.Text(label))
+ doc.note_autofootnote_ref(footnote_ref)
+ return footnote_ref
+
+ def make_footnote(doc: nodes.document, label: str, uri: str) -> nodes.footnote:
+ """Create a footnote node with children"""
+ footnote = nodes.footnote(uri)
+ para = nodes.paragraph()
+ para.append(nodes.Text(uri))
+ footnote.append(para)
+ footnote.insert(0, nodes.label('', label))
+ doc.note_autofootnote(footnote)
+ return footnote
+
+ def footnote_spot(tree: nodes.document) -> tuple[Element, int]:
+ """Find or create a spot to place footnotes.
+
+ The function returns the tuple (parent, index).
+ """
+ # The code uses the following heuristic:
+ # a) place them after the last existing footnote
+ # b) place them after an (empty) Footnotes rubric
+ # c) create an empty Footnotes rubric at the end of the document
+ fns = list(tree.findall(nodes.footnote))
+ if fns:
+ fn = fns[-1]
+ return fn.parent, fn.parent.index(fn) + 1
+ for node in tree.findall(nodes.rubric):
+ if len(node) == 1 and node.astext() == FOOTNOTES_RUBRIC_NAME:
+ return node.parent, node.parent.index(node) + 1
+ doc = next(tree.findall(nodes.document))
+ rub = nodes.rubric()
+ rub.append(nodes.Text(FOOTNOTES_RUBRIC_NAME))
+ doc.append(rub)
+ return doc, doc.index(rub) + 1
+
+ if show_urls == 'no':
+ return
+ if show_urls == 'footnote':
+ doc = next(tree.findall(nodes.document))
+ fn_spot, fn_idx = footnote_spot(tree)
+ nr = 1
+ for node in list(tree.findall(nodes.reference)):
+ uri = node.get('refuri', '')
+ if uri.startswith(('http:', 'https:', 'ftp:')) and uri not in node.astext():
+ idx = node.parent.index(node) + 1
+ if show_urls == 'inline':
+ uri = self.link_target_template % {'uri': uri}
+ link = nodes.inline(uri, uri)
+ link['classes'].append(self.css_link_target_class)
+ node.parent.insert(idx, link)
+ elif show_urls == 'footnote':
+ label = FOOTNOTE_LABEL_TEMPLATE % nr
+ nr += 1
+ footnote_ref = make_footnote_ref(doc, label)
+ node.parent.insert(idx, footnote_ref)
+ footnote = make_footnote(doc, label, uri)
+ fn_spot.insert(fn_idx, footnote)
+ footnote_ref['refid'] = footnote['ids'][0]
+ footnote.add_backref(footnote_ref['ids'][0])
+ fn_idx += 1
+
+ def write_doc(self, docname: str, doctree: nodes.document) -> None:
"""Write one document file.
This method is overwritten in order to fix fragment identifiers
and to add visible external links.
"""
- pass
+ self.fix_ids(doctree)
+ self.add_visible_links(doctree, self.config.epub_show_urls)
+ super().write_doc(docname, doctree)
- def fix_genindex(self, tree: list[tuple[str, list[tuple[str, Any]]]]
- ) ->None:
+ def fix_genindex(self, tree: list[tuple[str, list[tuple[str, Any]]]]) -> None:
"""Fix href attributes for genindex pages."""
- pass
-
- def is_vector_graphics(self, filename: str) ->bool:
+ # XXX: modifies tree inline
+ # Logic modeled from themes/basic/genindex.html
+ for _key, columns in tree:
+ for _entryname, (links, subitems, _key) in columns:
+ for (i, (ismain, link)) in enumerate(links):
+ m = self.refuri_re.match(link)
+ if m:
+ links[i] = (ismain,
+ self.fix_fragment(m.group(1), m.group(2)))
+ for _subentryname, subentrylinks in subitems:
+ for (i, (ismain, link)) in enumerate(subentrylinks):
+ m = self.refuri_re.match(link)
+ if m:
+ subentrylinks[i] = (ismain,
+ self.fix_fragment(m.group(1), m.group(2)))
+
+ def is_vector_graphics(self, filename: str) -> bool:
"""Does the filename extension indicate a vector graphic format?"""
- pass
+ ext = path.splitext(filename)[-1]
+ return ext in VECTOR_GRAPHICS_EXTENSIONS
- def copy_image_files_pil(self) ->None:
+ def copy_image_files_pil(self) -> None:
"""Copy images using Pillow, the Python Imaging Library.
The method tries to read and write the files with Pillow, converting
the format and resizing the image if necessary/possible.
"""
- pass
-
- def copy_image_files(self) ->None:
+ ensuredir(path.join(self.outdir, self.imagedir))
+ for src in status_iterator(self.images, __('copying images... '), "brown",
+ len(self.images), self.app.verbosity):
+ dest = self.images[src]
+ try:
+ img = Image.open(path.join(self.srcdir, src))
+ except OSError:
+ if not self.is_vector_graphics(src):
+ logger.warning(__('cannot read image file %r: copying it instead'),
+ path.join(self.srcdir, src))
+ try:
+ copyfile(
+ self.srcdir / src,
+ self.outdir / self.imagedir / dest,
+ force=True,
+ )
+ except OSError as err:
+ logger.warning(__('cannot copy image file %r: %s'),
+ path.join(self.srcdir, src), err)
+ continue
+ if self.config.epub_fix_images:
+ if img.mode == 'P':
+ # See the Pillow documentation for Image.convert()
+ # https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.convert
+ img = img.convert()
+ if self.config.epub_max_image_width > 0:
+ (width, height) = img.size
+ nw = self.config.epub_max_image_width
+ if width > nw:
+ nh = round((height * nw) / width)
+ img = img.resize((nw, nh), Image.BICUBIC)
+ try:
+ img.save(path.join(self.outdir, self.imagedir, dest))
+ except OSError as err:
+ logger.warning(__('cannot write image file %r: %s'),
+ path.join(self.srcdir, src), err)
+
+ def copy_image_files(self) -> None:
"""Copy image files to destination directory.
This overwritten method can use Pillow to convert image files.
"""
+ if self.images:
+ if self.config.epub_fix_images or self.config.epub_max_image_width:
+ if not PILLOW_AVAILABLE:
+ logger.warning(__('Pillow not found - copying image files'))
+ super().copy_image_files()
+ else:
+ self.copy_image_files_pil()
+ else:
+ super().copy_image_files()
+
+ def copy_download_files(self) -> None:
pass
- def handle_page(self, pagename: str, addctx: dict[str, Any],
- templatename: str='page.html', outfilename: (str | None)=None,
- event_arg: Any=None) ->None:
+ def handle_page(
+ self,
+ pagename: str,
+ addctx: dict[str, Any],
+ templatename: str = 'page.html',
+ outfilename: str | None = None,
+ event_arg: Any = None,
+ ) -> None:
"""Create a rendered page.
This method is overwritten for genindex pages in order to fix href link
attributes.
"""
- pass
-
- def build_mimetype(self) ->None:
+ if pagename.startswith('genindex') and 'genindexentries' in addctx:
+ if not self.use_index:
+ return
+ self.fix_genindex(addctx['genindexentries'])
+ addctx['doctype'] = self.doctype
+ super().handle_page(pagename, addctx, templatename, outfilename, event_arg)
+
+ def build_mimetype(self) -> None:
"""Write the metainfo file mimetype."""
- pass
-
- def build_container(self, outname: str='META-INF/container.xml') ->None:
+ logger.info(__('writing mimetype file...'))
+ copyfile(
+ path.join(self.template_dir, 'mimetype'),
+ self.outdir / 'mimetype',
+ force=True,
+ )
+
+ def build_container(self, outname: str = 'META-INF/container.xml') -> None:
"""Write the metainfo file META-INF/container.xml."""
- pass
-
- def content_metadata(self) ->dict[str, Any]:
+ logger.info(__('writing META-INF/container.xml file...'))
+ outdir = self.outdir / 'META-INF'
+ ensuredir(outdir)
+ copyfile(
+ path.join(self.template_dir, 'container.xml'),
+ outdir / 'container.xml',
+ force=True,
+ )
+
+ def content_metadata(self) -> dict[str, Any]:
"""Create a dictionary with all metadata for the content.opf
file properly escaped.
"""
- pass
-
- def build_content(self) ->None:
+ if (source_date_epoch := os.getenv('SOURCE_DATE_EPOCH')) is not None:
+ time_tuple = time.gmtime(int(source_date_epoch))
+ else:
+ time_tuple = time.gmtime()
+
+ metadata: dict[str, Any] = {}
+ metadata['title'] = html.escape(self.config.epub_title)
+ metadata['author'] = html.escape(self.config.epub_author)
+ metadata['uid'] = html.escape(self.config.epub_uid)
+ metadata['lang'] = html.escape(self.config.epub_language)
+ metadata['publisher'] = html.escape(self.config.epub_publisher)
+ metadata['copyright'] = html.escape(self.config.epub_copyright)
+ metadata['scheme'] = html.escape(self.config.epub_scheme)
+ metadata['id'] = html.escape(self.config.epub_identifier)
+ metadata['date'] = html.escape(time.strftime('%Y-%m-%d', time_tuple))
+ metadata['manifest_items'] = []
+ metadata['spines'] = []
+ metadata['guides'] = []
+ return metadata
+
+ def build_content(self) -> None:
"""Write the metainfo file content.opf It contains bibliographic data,
a file list and the spine (the reading order).
"""
- pass
-
- def new_navpoint(self, node: dict[str, Any], level: int, incr: bool=True
- ) ->NavPoint:
+ logger.info(__('writing content.opf file...'))
+ metadata = self.content_metadata()
+
+ # files
+ self.files: list[str] = []
+ self.ignored_files = [
+ '.buildinfo',
+ 'mimetype',
+ 'content.opf',
+ 'toc.ncx',
+ 'META-INF/container.xml',
+ 'Thumbs.db',
+ 'ehthumbs.db',
+ '.DS_Store',
+ 'nav.xhtml',
+ self.config.epub_basename + '.epub',
+ *self.config.epub_exclude_files,
+ ]
+ if not self.use_index:
+ self.ignored_files.append('genindex' + self.out_suffix)
+ for root, dirs, files in os.walk(self.outdir):
+ dirs.sort()
+ for fn in sorted(files):
+ filename = relpath(path.join(root, fn), self.outdir)
+ if filename in self.ignored_files:
+ continue
+ ext = path.splitext(filename)[-1]
+ if ext not in self.media_types:
+ # we always have JS and potentially OpenSearch files, don't
+ # always warn about them
+ if ext not in ('.js', '.xml'):
+ logger.warning(__('unknown mimetype for %s, ignoring'), filename,
+ type='epub', subtype='unknown_project_files')
+ continue
+ filename = filename.replace(os.sep, '/')
+ item = ManifestItem(html.escape(quote(filename)),
+ html.escape(self.make_id(filename)),
+ html.escape(self.media_types[ext]))
+ metadata['manifest_items'].append(item)
+ self.files.append(filename)
+
+ # spine
+ spinefiles = set()
+ for refnode in self.refnodes:
+ if '#' in refnode['refuri']:
+ continue
+ if refnode['refuri'] in self.ignored_files:
+ continue
+ spine = Spine(html.escape(self.make_id(refnode['refuri'])), True)
+ metadata['spines'].append(spine)
+ spinefiles.add(refnode['refuri'])
+ for info in self.domain_indices:
+ spine = Spine(html.escape(self.make_id(info[0] + self.out_suffix)), True)
+ metadata['spines'].append(spine)
+ spinefiles.add(info[0] + self.out_suffix)
+ if self.use_index:
+ spine = Spine(html.escape(self.make_id('genindex' + self.out_suffix)), True)
+ metadata['spines'].append(spine)
+ spinefiles.add('genindex' + self.out_suffix)
+ # add auto generated files
+ for name in self.files:
+ if name not in spinefiles and name.endswith(self.out_suffix):
+ spine = Spine(html.escape(self.make_id(name)), False)
+ metadata['spines'].append(spine)
+
+ # add the optional cover
+ html_tmpl = None
+ if self.config.epub_cover:
+ image, html_tmpl = self.config.epub_cover
+ image = image.replace(os.sep, '/')
+ metadata['cover'] = html.escape(self.make_id(image))
+ if html_tmpl:
+ spine = Spine(html.escape(self.make_id(self.coverpage_name)), True)
+ metadata['spines'].insert(0, spine)
+ if self.coverpage_name not in self.files:
+ ext = path.splitext(self.coverpage_name)[-1]
+ self.files.append(self.coverpage_name)
+ item = ManifestItem(html.escape(self.coverpage_name),
+ html.escape(self.make_id(self.coverpage_name)),
+ html.escape(self.media_types[ext]))
+ metadata['manifest_items'].append(item)
+ ctx = {'image': html.escape(image), 'title': self.config.project}
+ self.handle_page(
+ path.splitext(self.coverpage_name)[0], ctx, html_tmpl)
+ spinefiles.add(self.coverpage_name)
+
+ auto_add_cover = True
+ auto_add_toc = True
+ if self.config.epub_guide:
+ for type, uri, title in self.config.epub_guide:
+ file = uri.split('#')[0]
+ if file not in self.files:
+ self.files.append(file)
+ if type == 'cover':
+ auto_add_cover = False
+ if type == 'toc':
+ auto_add_toc = False
+ metadata['guides'].append(Guide(html.escape(type),
+ html.escape(title),
+ html.escape(uri)))
+ if auto_add_cover and html_tmpl:
+ metadata['guides'].append(Guide('cover',
+ self.guide_titles['cover'],
+ html.escape(self.coverpage_name)))
+ if auto_add_toc and self.refnodes:
+ metadata['guides'].append(Guide('toc',
+ self.guide_titles['toc'],
+ html.escape(self.refnodes[0]['refuri'])))
+
+ # write the project file
+ copy_asset_file(
+ path.join(self.template_dir, 'content.opf.jinja'),
+ self.outdir,
+ context=metadata,
+ force=True,
+ )
+
+ def new_navpoint(self, node: dict[str, Any], level: int, incr: bool = True) -> NavPoint:
"""Create a new entry in the toc from the node at given level."""
- pass
-
- def build_navpoints(self, nodes: list[dict[str, Any]]) ->list[NavPoint]:
+ # XXX Modifies the node
+ if incr:
+ self.playorder += 1
+ self.tocid += 1
+ return NavPoint('navPoint%d' % self.tocid, self.playorder,
+ node['text'], node['refuri'], [])
+
+ def build_navpoints(self, nodes: list[dict[str, Any]]) -> list[NavPoint]:
"""Create the toc navigation structure.
Subelements of a node are nested inside the navpoint. For nested nodes
the parent node is reinserted in the subnav.
"""
- pass
-
- def toc_metadata(self, level: int, navpoints: list[NavPoint]) ->dict[
- str, Any]:
+ navstack: list[NavPoint] = []
+ navstack.append(NavPoint('dummy', 0, '', '', []))
+ level = 0
+ lastnode = None
+ for node in nodes:
+ if not node['text']:
+ continue
+ file = node['refuri'].split('#')[0]
+ if file in self.ignored_files:
+ continue
+ if node['level'] > self.config.epub_tocdepth:
+ continue
+ if node['level'] == level:
+ navpoint = self.new_navpoint(node, level)
+ navstack.pop()
+ navstack[-1].children.append(navpoint)
+ navstack.append(navpoint)
+ elif node['level'] == level + 1:
+ level += 1
+ if lastnode and self.config.epub_tocdup:
+ # Insert starting point in subtoc with same playOrder
+ navstack[-1].children.append(self.new_navpoint(lastnode, level, False))
+ navpoint = self.new_navpoint(node, level)
+ navstack[-1].children.append(navpoint)
+ navstack.append(navpoint)
+ elif node['level'] < level:
+ while node['level'] < len(navstack):
+ navstack.pop()
+ level = node['level']
+ navpoint = self.new_navpoint(node, level)
+ navstack[-1].children.append(navpoint)
+ navstack.append(navpoint)
+ else:
+ raise
+ lastnode = node
+
+ return navstack[0].children
+
+ def toc_metadata(self, level: int, navpoints: list[NavPoint]) -> dict[str, Any]:
"""Create a dictionary with all metadata for the toc.ncx file
properly escaped.
"""
- pass
-
- def build_toc(self) ->None:
+ metadata: dict[str, Any] = {}
+ metadata['uid'] = self.config.epub_uid
+ metadata['title'] = html.escape(self.config.epub_title)
+ metadata['level'] = level
+ metadata['navpoints'] = navpoints
+ return metadata
+
+ def build_toc(self) -> None:
"""Write the metainfo file toc.ncx."""
- pass
-
- def build_epub(self) ->None:
+ logger.info(__('writing toc.ncx file...'))
+
+ if self.config.epub_tocscope == 'default':
+ doctree = self.env.get_and_resolve_doctree(self.config.root_doc,
+ self, prune_toctrees=False,
+ includehidden=False)
+ refnodes = self.get_refnodes(doctree, [])
+ self.toc_add_files(refnodes)
+ else:
+ # 'includehidden'
+ refnodes = self.refnodes
+ self.check_refnodes(refnodes)
+ navpoints = self.build_navpoints(refnodes)
+ level = max(item['level'] for item in self.refnodes)
+ level = min(level, self.config.epub_tocdepth)
+ copy_asset_file(
+ path.join(self.template_dir, 'toc.ncx.jinja'),
+ self.outdir,
+ context=self.toc_metadata(level, navpoints),
+ force=True,
+ )
+
+ def build_epub(self) -> None:
"""Write the epub file.
It is a zip file with the mimetype file stored uncompressed as the first
entry.
"""
- pass
+ outname = self.config.epub_basename + '.epub'
+ logger.info(__('writing %s file...'), outname)
+ epub_filename = path.join(self.outdir, outname)
+ with ZipFile(epub_filename, 'w', ZIP_DEFLATED) as epub:
+ epub.write(path.join(self.outdir, 'mimetype'), 'mimetype', ZIP_STORED)
+ for filename in ('META-INF/container.xml', 'content.opf', 'toc.ncx'):
+ epub.write(path.join(self.outdir, filename), filename, ZIP_DEFLATED)
+ for filename in self.files:
+ epub.write(path.join(self.outdir, filename), filename, ZIP_DEFLATED)
diff --git a/sphinx/builders/changes.py b/sphinx/builders/changes.py
index 978bdc95a..afc5f064b 100644
--- a/sphinx/builders/changes.py
+++ b/sphinx/builders/changes.py
@@ -1,8 +1,11 @@
"""Changelog builder."""
+
from __future__ import annotations
+
import html
from os import path
from typing import TYPE_CHECKING, Any, cast
+
from sphinx import package_dir
from sphinx.builders import Builder
from sphinx.domains.changeset import ChangeSetDomain
@@ -12,9 +15,11 @@ from sphinx.util import logging
from sphinx.util.console import bold
from sphinx.util.fileutil import copy_asset_file
from sphinx.util.osutil import ensuredir, os_path
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
@@ -22,7 +27,148 @@ class ChangesBuilder(Builder):
"""
Write a summary with all versionadded/changed/deprecated/removed directives.
"""
+
name = 'changes'
epilog = __('The overview file is in %(outdir)s.')
- typemap = {'versionadded': 'added', 'versionchanged': 'changed',
- 'deprecated': 'deprecated', 'versionremoved': 'removed'}
+
+ def init(self) -> None:
+ self.create_template_bridge()
+ theme_factory = HTMLThemeFactory(self.app)
+ self.theme = theme_factory.create('default')
+ self.templates.init(self, self.theme)
+
+ def get_outdated_docs(self) -> str:
+ return str(self.outdir)
+
+ typemap = {
+ 'versionadded': 'added',
+ 'versionchanged': 'changed',
+ 'deprecated': 'deprecated',
+ 'versionremoved': 'removed',
+ }
+
+ def write(self, *ignored: Any) -> None:
+ version = self.config.version
+ domain = cast(ChangeSetDomain, self.env.get_domain('changeset'))
+ libchanges: dict[str, list[tuple[str, str, int]]] = {}
+ apichanges: list[tuple[str, str, int]] = []
+ otherchanges: dict[tuple[str, str], list[tuple[str, str, int]]] = {}
+
+ changesets = domain.get_changesets_for(version)
+ if not changesets:
+ logger.info(bold(__('no changes in version %s.')), version)
+ return
+ logger.info(bold(__('writing summary file...')))
+ for changeset in changesets:
+ if isinstance(changeset.descname, tuple):
+ descname = changeset.descname[0]
+ else:
+ descname = changeset.descname
+ ttext = self.typemap[changeset.type]
+ context = changeset.content.replace('\n', ' ')
+ if descname and changeset.docname.startswith('c-api'):
+ if context:
+ entry = f'<b>{descname}</b>: <i>{ttext}:</i> {context}'
+ else:
+ entry = f'<b>{descname}</b>: <i>{ttext}</i>.'
+ apichanges.append((entry, changeset.docname, changeset.lineno))
+ elif descname or changeset.module:
+ module = changeset.module or _('Builtins')
+ if not descname:
+ descname = _('Module level')
+ if context:
+ entry = f'<b>{descname}</b>: <i>{ttext}:</i> {context}'
+ else:
+ entry = f'<b>{descname}</b>: <i>{ttext}</i>.'
+ libchanges.setdefault(module, []).append((entry, changeset.docname,
+ changeset.lineno))
+ else:
+ if not context:
+ continue
+ entry = f'<i>{ttext.capitalize()}:</i> {context}'
+ title = self.env.titles[changeset.docname].astext()
+ otherchanges.setdefault((changeset.docname, title), []).append(
+ (entry, changeset.docname, changeset.lineno))
+
+ ctx = {
+ 'project': self.config.project,
+ 'version': version,
+ 'docstitle': self.config.html_title,
+ 'shorttitle': self.config.html_short_title,
+ 'libchanges': sorted(libchanges.items()),
+ 'apichanges': sorted(apichanges),
+ 'otherchanges': sorted(otherchanges.items()),
+ 'show_copyright': self.config.html_show_copyright,
+ 'show_sphinx': self.config.html_show_sphinx,
+ }
+ with open(path.join(self.outdir, 'index.html'), 'w', encoding='utf8') as f:
+ f.write(self.templates.render('changes/frameset.html', ctx))
+ with open(path.join(self.outdir, 'changes.html'), 'w', encoding='utf8') as f:
+ f.write(self.templates.render('changes/versionchanges.html', ctx))
+
+ hltext = ['.. versionadded:: %s' % version,
+ '.. versionchanged:: %s' % version,
+ '.. deprecated:: %s' % version,
+ '.. versionremoved:: %s' % version,
+ ]
+
+ def hl(no: int, line: str) -> str:
+ line = '<a name="L%s"> </a>' % no + html.escape(line)
+ for x in hltext:
+ if x in line:
+ line = '<span class="hl">%s</span>' % line
+ break
+ return line
+
+ logger.info(bold(__('copying source files...')))
+ for docname in self.env.all_docs:
+ with open(self.env.doc2path(docname),
+ encoding=self.env.config.source_encoding) as f:
+ try:
+ lines = f.readlines()
+ except UnicodeDecodeError:
+ logger.warning(__('could not read %r for changelog creation'), docname)
+ continue
+ targetfn = path.join(self.outdir, 'rst', os_path(docname)) + '.html'
+ ensuredir(path.dirname(targetfn))
+ with open(targetfn, 'w', encoding='utf-8') as f:
+ text = ''.join(hl(i + 1, line) for (i, line) in enumerate(lines))
+ ctx = {
+ 'filename': str(self.env.doc2path(docname, False)),
+ 'text': text,
+ }
+ f.write(self.templates.render('changes/rstsource.html', ctx))
+ themectx = {'theme_' + key: val for (key, val) in
+ self.theme.get_options({}).items()}
+ copy_asset_file(
+ path.join(package_dir, 'themes', 'default', 'static', 'default.css.jinja'),
+ self.outdir,
+ context=themectx,
+ renderer=self.templates,
+ force=True,
+ )
+ copy_asset_file(
+ path.join(package_dir, 'themes', 'basic', 'static', 'basic.css'),
+ self.outdir / 'basic.css',
+ force=True,
+ )
+
+ def hl(self, text: str, version: str) -> str:
+ text = html.escape(text)
+ for directive in ('versionchanged', 'versionadded', 'deprecated', 'versionremoved'):
+ text = text.replace(f'.. {directive}:: {version}',
+ f'<b>.. {directive}:: {version}</b>')
+ return text
+
+ def finish(self) -> None:
+ pass
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(ChangesBuilder)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/dirhtml.py b/sphinx/builders/dirhtml.py
index b59f4ec82..dbfced3a7 100644
--- a/sphinx/builders/dirhtml.py
+++ b/sphinx/builders/dirhtml.py
@@ -1,13 +1,18 @@
"""Directory HTML builders."""
+
from __future__ import annotations
+
from os import path
from typing import TYPE_CHECKING
+
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.util import logging
from sphinx.util.osutil import SEP, os_path
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
@@ -17,4 +22,34 @@ class DirectoryHTMLBuilder(StandaloneHTMLBuilder):
a directory given by their pagename, so that generated URLs don't have
``.html`` in them.
"""
+
name = 'dirhtml'
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ if docname == 'index':
+ return ''
+ if docname.endswith(SEP + 'index'):
+ return docname[:-5] # up to sep
+ return docname + SEP
+
+ def get_outfilename(self, pagename: str) -> str:
+ if pagename == 'index' or pagename.endswith(SEP + 'index'):
+ outfilename = path.join(self.outdir, os_path(pagename) +
+ self.out_suffix)
+ else:
+ outfilename = path.join(self.outdir, os_path(pagename),
+ 'index' + self.out_suffix)
+
+ return outfilename
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.setup_extension('sphinx.builders.html')
+
+ app.add_builder(DirectoryHTMLBuilder)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/dummy.py b/sphinx/builders/dummy.py
index f7d1fb7c5..05b7e5687 100644
--- a/sphinx/builders/dummy.py
+++ b/sphinx/builders/dummy.py
@@ -1,10 +1,15 @@
"""Do syntax checks, but no writing."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING
+
from sphinx.builders import Builder
from sphinx.locale import __
+
if TYPE_CHECKING:
from docutils import nodes
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
@@ -12,4 +17,33 @@ if TYPE_CHECKING:
class DummyBuilder(Builder):
name = 'dummy'
epilog = __('The dummy builder generates no files.')
+
allow_parallel = True
+
+ def init(self) -> None:
+ pass
+
+ def get_outdated_docs(self) -> set[str]:
+ return self.env.found_docs
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ return ''
+
+ def prepare_writing(self, docnames: set[str]) -> None:
+ pass
+
+ def write_doc(self, docname: str, doctree: nodes.document) -> None:
+ pass
+
+ def finish(self) -> None:
+ pass
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(DummyBuilder)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/epub3.py b/sphinx/builders/epub3.py
index aecaf3dbc..004821b6e 100644
--- a/sphinx/builders/epub3.py
+++ b/sphinx/builders/epub3.py
@@ -2,13 +2,16 @@
Originally derived from epub.py.
"""
+
from __future__ import annotations
+
import html
import os
import re
import time
from os import path
from typing import TYPE_CHECKING, Any, NamedTuple
+
from sphinx import package_dir
from sphinx.builders import _epub_base
from sphinx.config import ENUM, Config
@@ -16,9 +19,11 @@ from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.fileutil import copy_asset_file
from sphinx.util.osutil import make_filename
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
@@ -28,18 +33,38 @@ class NavPoint(NamedTuple):
children: list[NavPoint]
-PAGE_PROGRESSION_DIRECTIONS = {'horizontal': 'ltr', 'vertical': 'rtl'}
-IBOOK_SCROLL_AXIS = {'horizontal': 'vertical', 'vertical': 'horizontal'}
-THEME_WRITING_MODES = {'vertical': 'vertical-rl', 'horizontal': 'horizontal-tb'
- }
-DOCTYPE = '<!DOCTYPE html>'
+# writing modes
+PAGE_PROGRESSION_DIRECTIONS = {
+ 'horizontal': 'ltr',
+ 'vertical': 'rtl',
+}
+IBOOK_SCROLL_AXIS = {
+ 'horizontal': 'vertical',
+ 'vertical': 'horizontal',
+}
+THEME_WRITING_MODES = {
+ 'vertical': 'vertical-rl',
+ 'horizontal': 'horizontal-tb',
+}
+
+DOCTYPE = '''<!DOCTYPE html>'''
+
HTML_TAG = (
- '<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">'
- )
+ '<html xmlns="http://www.w3.org/1999/xhtml" '
+ 'xmlns:epub="http://www.idpf.org/2007/ops">'
+)
+
+# https://www.w3.org/TR/REC-xml/#NT-Name
_xml_name_start_char = (
- ':|[A-Z]|_|[a-z]|[À-Ö]|[Ø-ö]|[ø-˿]|[Ͱ-ͽ]|[Ϳ-\u1fff]|[\u200c-\u200d]|[⁰-\u218f]|[Ⰰ-\u2fef]|[、-\ud7ff]|[豈-﷏]|[ﷰ-�]|[𐀀-\U000effff]'
- )
-_xml_name_char = _xml_name_start_char + '\\-|\\.|[0-9]|·|[̀-ͯ]|[‿-⁀]'
+ ':|[A-Z]|_|[a-z]|[\u00C0-\u00D6]'
+ '|[\u00D8-\u00F6]|[\u00F8-\u02FF]|[\u0370-\u037D]'
+ '|[\u037F-\u1FFF]|[\u200C-\u200D]|[\u2070-\u218F]'
+ '|[\u2C00-\u2FEF]|[\u3001-\uD7FF]|[\uF900-\uFDCF]'
+ '|[\uFDF0-\uFFFD]|[\U00010000-\U000EFFFF]'
+)
+_xml_name_char = (
+ _xml_name_start_char + r'\-|\.' '|[0-9]|\u00B7|[\u0300-\u036F]|[\u203F-\u2040]'
+)
_XML_NAME_PATTERN = re.compile(f'({_xml_name_start_char})({_xml_name_char})*')
@@ -51,25 +76,58 @@ class Epub3Builder(_epub_base.EpubBuilder):
and META-INF/container.xml. Afterwards, all necessary files are zipped to
an epub file.
"""
+
name = 'epub'
epilog = __('The ePub file is in %(outdir)s.')
+
supported_remote_images = False
template_dir = path.join(package_dir, 'templates', 'epub3')
doctype = DOCTYPE
html_tag = HTML_TAG
use_meta_charset = True
- def handle_finish(self) ->None:
+ # Finish by building the epub file
+ def handle_finish(self) -> None:
"""Create the metainfo files and finally the epub."""
- pass
+ self.get_toc()
+ self.build_mimetype()
+ self.build_container()
+ self.build_content()
+ self.build_navigation_doc()
+ self.build_toc()
+ self.build_epub()
- def content_metadata(self) ->dict[str, Any]:
+ def content_metadata(self) -> dict[str, Any]:
"""Create a dictionary with all metadata for the content.opf
file properly escaped.
"""
- pass
+ writing_mode = self.config.epub_writing_mode
+
+ if (source_date_epoch := os.getenv('SOURCE_DATE_EPOCH')) is not None:
+ time_tuple = time.gmtime(int(source_date_epoch))
+ else:
+ time_tuple = time.gmtime()
+
+ metadata = super().content_metadata()
+ metadata['description'] = html.escape(self.config.epub_description)
+ metadata['contributor'] = html.escape(self.config.epub_contributor)
+ metadata['page_progression_direction'] = PAGE_PROGRESSION_DIRECTIONS.get(writing_mode)
+ metadata['ibook_scroll_axis'] = IBOOK_SCROLL_AXIS.get(writing_mode)
+ metadata['date'] = html.escape(time.strftime("%Y-%m-%dT%H:%M:%SZ", time_tuple))
+ metadata['version'] = html.escape(self.config.version)
+ metadata['epub_version'] = self.config.epub_version
+ return metadata
+
+ def prepare_writing(self, docnames: set[str]) -> None:
+ super().prepare_writing(docnames)
+
+ writing_mode = self.config.epub_writing_mode
+ self.globalcontext['theme_writing_mode'] = THEME_WRITING_MODES.get(writing_mode)
+ self.globalcontext['html_tag'] = self.html_tag
+ self.globalcontext['use_meta_charset'] = self.use_meta_charset
+ self.globalcontext['skip_ua_compatible'] = True
- def build_navlist(self, navnodes: list[dict[str, Any]]) ->list[NavPoint]:
+ def build_navlist(self, navnodes: list[dict[str, Any]]) -> list[NavPoint]:
"""Create the toc navigation structure.
This method is almost same as build_navpoints method in epub.py.
@@ -79,20 +137,171 @@ class Epub3Builder(_epub_base.EpubBuilder):
The difference from build_navpoints method is templates which are used
when generating navigation documents.
"""
- pass
+ navstack: list[NavPoint] = []
+ navstack.append(NavPoint('', '', []))
+ level = 0
+ for node in navnodes:
+ if not node['text']:
+ continue
+ file = node['refuri'].split('#')[0]
+ if file in self.ignored_files:
+ continue
+ if node['level'] > self.config.epub_tocdepth:
+ continue
- def navigation_doc_metadata(self, navlist: list[NavPoint]) ->dict[str, Any
- ]:
+ navpoint = NavPoint(node['text'], node['refuri'], [])
+ if node['level'] == level:
+ navstack.pop()
+ navstack[-1].children.append(navpoint)
+ navstack.append(navpoint)
+ elif node['level'] == level + 1:
+ level += 1
+ navstack[-1].children.append(navpoint)
+ navstack.append(navpoint)
+ elif node['level'] < level:
+ while node['level'] < len(navstack):
+ navstack.pop()
+ level = node['level']
+ navstack[-1].children.append(navpoint)
+ navstack.append(navpoint)
+ else:
+ unreachable = 'Should never reach here. It might be a bug.'
+ raise RuntimeError(unreachable)
+
+ return navstack[0].children
+
+ def navigation_doc_metadata(self, navlist: list[NavPoint]) -> dict[str, Any]:
"""Create a dictionary with all metadata for the nav.xhtml file
properly escaped.
"""
- pass
+ return {
+ 'lang': html.escape(self.config.epub_language),
+ 'toc_locale': html.escape(self.guide_titles['toc']),
+ 'navlist': navlist,
+ }
- def build_navigation_doc(self) ->None:
+ def build_navigation_doc(self) -> None:
"""Write the metainfo file nav.xhtml."""
- pass
+ logger.info(__('writing nav.xhtml file...'))
+
+ if self.config.epub_tocscope == 'default':
+ doctree = self.env.get_and_resolve_doctree(
+ self.config.root_doc, self,
+ prune_toctrees=False, includehidden=False)
+ refnodes = self.get_refnodes(doctree, [])
+ self.toc_add_files(refnodes)
+ else:
+ # 'includehidden'
+ refnodes = self.refnodes
+ navlist = self.build_navlist(refnodes)
+ copy_asset_file(
+ path.join(self.template_dir, 'nav.xhtml.jinja'),
+ self.outdir,
+ context=self.navigation_doc_metadata(navlist),
+ force=True,
+ )
+
+ # Add nav.xhtml to epub file
+ if 'nav.xhtml' not in self.files:
+ self.files.append('nav.xhtml')
+
+
+def validate_config_values(app: Sphinx) -> None:
+ if app.builder.name != 'epub':
+ return
+
+ # <package> lang attribute, dc:language
+ if not app.config.epub_language:
+ logger.warning(__('conf value "epub_language" (or "language") '
+ 'should not be empty for EPUB3'))
+ # <package> unique-identifier attribute
+ if not _XML_NAME_PATTERN.match(app.config.epub_uid):
+ logger.warning(__('conf value "epub_uid" should be XML NAME for EPUB3'))
+ # dc:title
+ if not app.config.epub_title:
+ logger.warning(__('conf value "epub_title" (or "html_title") '
+ 'should not be empty for EPUB3'))
+ # dc:creator
+ if not app.config.epub_author:
+ logger.warning(__('conf value "epub_author" should not be empty for EPUB3'))
+ # dc:contributor
+ if not app.config.epub_contributor:
+ logger.warning(__('conf value "epub_contributor" should not be empty for EPUB3'))
+ # dc:description
+ if not app.config.epub_description:
+ logger.warning(__('conf value "epub_description" should not be empty for EPUB3'))
+ # dc:publisher
+ if not app.config.epub_publisher:
+ logger.warning(__('conf value "epub_publisher" should not be empty for EPUB3'))
+ # dc:rights
+ if not app.config.epub_copyright:
+ logger.warning(__('conf value "epub_copyright" (or "copyright")'
+ 'should not be empty for EPUB3'))
+ # dc:identifier
+ if not app.config.epub_identifier:
+ logger.warning(__('conf value "epub_identifier" should not be empty for EPUB3'))
+ # meta ibooks:version
+ if not app.config.version:
+ logger.warning(__('conf value "version" should not be empty for EPUB3'))
-def convert_epub_css_files(app: Sphinx, config: Config) ->None:
+def convert_epub_css_files(app: Sphinx, config: Config) -> None:
"""Convert string styled epub_css_files to tuple styled one."""
- pass
+ epub_css_files: list[tuple[str, dict[str, Any]]] = []
+ for entry in config.epub_css_files:
+ if isinstance(entry, str):
+ epub_css_files.append((entry, {}))
+ else:
+ try:
+ filename, attrs = entry
+ epub_css_files.append((filename, attrs))
+ except Exception:
+ logger.warning(__('invalid css_file: %r, ignored'), entry)
+ continue
+
+ config.epub_css_files = epub_css_files
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(Epub3Builder)
+
+ # config values
+ app.add_config_value('epub_basename', lambda self: make_filename(self.project), '')
+ app.add_config_value('epub_version', 3.0, 'epub') # experimental
+ app.add_config_value('epub_theme', 'epub', 'epub')
+ app.add_config_value('epub_theme_options', {}, 'epub')
+ app.add_config_value('epub_title', lambda self: self.project, 'epub')
+ app.add_config_value('epub_author', lambda self: self.author, 'epub')
+ app.add_config_value('epub_language', lambda self: self.language or 'en', 'epub')
+ app.add_config_value('epub_publisher', lambda self: self.author, 'epub')
+ app.add_config_value('epub_copyright', lambda self: self.copyright, 'epub')
+ app.add_config_value('epub_identifier', 'unknown', 'epub')
+ app.add_config_value('epub_scheme', 'unknown', 'epub')
+ app.add_config_value('epub_uid', 'unknown', 'env')
+ app.add_config_value('epub_cover', (), 'env')
+ app.add_config_value('epub_guide', (), 'env')
+ app.add_config_value('epub_pre_files', [], 'env')
+ app.add_config_value('epub_post_files', [], 'env')
+ app.add_config_value('epub_css_files', lambda config: config.html_css_files, 'epub')
+ app.add_config_value('epub_exclude_files', [], 'env')
+ app.add_config_value('epub_tocdepth', 3, 'env')
+ app.add_config_value('epub_tocdup', True, 'env')
+ app.add_config_value('epub_tocscope', 'default', 'env')
+ app.add_config_value('epub_fix_images', False, 'env')
+ app.add_config_value('epub_max_image_width', 0, 'env')
+ app.add_config_value('epub_show_urls', 'inline', 'epub')
+ app.add_config_value('epub_use_index', lambda self: self.html_use_index, 'epub')
+ app.add_config_value('epub_description', 'unknown', 'epub')
+ app.add_config_value('epub_contributor', 'unknown', 'epub')
+ app.add_config_value('epub_writing_mode', 'horizontal', 'epub',
+ ENUM('horizontal', 'vertical'))
+
+ # event handlers
+ app.connect('config-inited', convert_epub_css_files, priority=800)
+ app.connect('builder-inited', validate_config_values)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/gettext.py b/sphinx/builders/gettext.py
index 4b4b24de7..8427fcbb4 100644
--- a/sphinx/builders/gettext.py
+++ b/sphinx/builders/gettext.py
@@ -1,5 +1,7 @@
"""The MessageCatalogBuilder class."""
+
from __future__ import annotations
+
import operator
import time
from codecs import open
@@ -8,7 +10,9 @@ from os import getenv, path, walk
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal
from uuid import uuid4
+
from docutils import nodes
+
from sphinx import addnodes, package_dir
from sphinx.builders import Builder
from sphinx.errors import ThemeError
@@ -22,22 +26,26 @@ from sphinx.util.nodes import extract_messages, traverse_translatable_index
from sphinx.util.osutil import canon_path, ensuredir, relpath
from sphinx.util.tags import Tags
from sphinx.util.template import SphinxRenderer
+
if TYPE_CHECKING:
import os
from collections.abc import Iterable, Iterator, Sequence
+
from docutils.nodes import Element
+
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata
+
DEFAULT_TEMPLATE_PATH = Path(package_dir, 'templates', 'gettext')
+
logger = logging.getLogger(__name__)
class Message:
"""An entry of translatable message."""
- def __init__(self, text: str, locations: list[tuple[str, int]], uuids:
- list[str]) ->None:
+ def __init__(self, text: str, locations: list[tuple[str, int]], uuids: list[str]) -> None:
self.text = text
self.locations = locations
self.uuids = uuids
@@ -46,14 +54,29 @@ class Message:
class Catalog:
"""Catalog of translatable messages."""
- def __init__(self) ->None:
- self.messages: list[str] = []
+ def __init__(self) -> None:
+ self.messages: list[str] = [] # retain insertion order
+
+ # msgid -> file, line, uid
self.metadata: dict[str, list[tuple[str, int, str]]] = {}
- def __iter__(self) ->Iterator[Message]:
+ def add(self, msg: str, origin: Element | MsgOrigin) -> None:
+ if not hasattr(origin, 'uid'):
+ # Nodes that are replicated like todo don't have a uid,
+ # however i18n is also unnecessary.
+ return
+ if msg not in self.metadata: # faster lookup in hash
+ self.messages.append(msg)
+ self.metadata[msg] = []
+ line = origin.line
+ if line is None:
+ line = -1
+ self.metadata[msg].append((origin.source, line, origin.uid)) # type: ignore[arg-type]
+
+ def __iter__(self) -> Iterator[Message]:
for message in self.messages:
- positions = sorted({(source, line) for source, line, uuid in
- self.metadata[message]})
+ positions = sorted({(source, line) for source, line, uuid
+ in self.metadata[message]})
uuids = [uuid for source, line, uuid in self.metadata[message]]
yield Message(message, positions, uuids)
@@ -63,29 +86,39 @@ class MsgOrigin:
Origin holder for Catalog message origin.
"""
- def __init__(self, source: str, line: int) ->None:
+ def __init__(self, source: str, line: int) -> None:
self.source = source
self.line = line
self.uid = uuid4().hex
class GettextRenderer(SphinxRenderer):
-
- def __init__(self, template_path: (Sequence[str | os.PathLike[str]] |
- None)=None, outdir: (str | os.PathLike[str] | None)=None) ->None:
+ def __init__(
+ self, template_path: Sequence[str | os.PathLike[str]] | None = None,
+ outdir: str | os.PathLike[str] | None = None,
+ ) -> None:
self.outdir = outdir
if template_path is None:
super().__init__([DEFAULT_TEMPLATE_PATH])
else:
super().__init__([*template_path, DEFAULT_TEMPLATE_PATH])
- def escape(s: str) ->str:
- s = s.replace('\\', '\\\\')
- s = s.replace('"', '\\"')
+ def escape(s: str) -> str:
+ s = s.replace('\\', r'\\')
+ s = s.replace('"', r'\"')
return s.replace('\n', '\\n"\n"')
+
+ # use texescape as escape filter
self.env.filters['e'] = escape
self.env.filters['escape'] = escape
+ def render(self, filename: str, context: dict[str, Any]) -> str:
+ def _relpath(s: str) -> str:
+ return canon_path(relpath(s, self.outdir))
+
+ context['relpath'] = _relpath
+ return super().render(filename, context)
+
class I18nTags(Tags):
"""Dummy tags module for I18nBuilder.
@@ -94,31 +127,206 @@ class I18nTags(Tags):
this class always returns ``True`` regardless the defined tags.
"""
+ def eval_condition(self, condition: Any) -> bool:
+ return True
+
class I18nBuilder(Builder):
"""
General i18n builder.
"""
+
name = 'i18n'
versioning_method = 'text'
use_message_catalog = False
+ def init(self) -> None:
+ super().init()
+ self.env.set_versioning_method(self.versioning_method,
+ self.env.config.gettext_uuid)
+ self.tags = I18nTags()
+ self.catalogs: defaultdict[str, Catalog] = defaultdict(Catalog)
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ return ''
+
+ def get_outdated_docs(self) -> set[str]:
+ return self.env.found_docs
+
+ def prepare_writing(self, docnames: set[str]) -> None:
+ return
+
+ def compile_catalogs(self, catalogs: set[CatalogInfo], message: str) -> None:
+ return
+
+ def write_doc(self, docname: str, doctree: nodes.document) -> None:
+ catalog = self.catalogs[docname_to_domain(docname, self.config.gettext_compact)]
+
+ for toctree in self.env.tocs[docname].findall(addnodes.toctree):
+ for node, msg in extract_messages(toctree):
+ node.uid = '' # type: ignore[attr-defined] # Hack UUID model
+ catalog.add(msg, node)
+ for node, msg in extract_messages(doctree):
+ # Do not extract messages from within substitution definitions.
+ if not _is_node_in_substitution_definition(node):
+ catalog.add(msg, node)
+
+ if 'index' in self.env.config.gettext_additional_targets:
+ # Extract translatable messages from index entries.
+ for node, entries in traverse_translatable_index(doctree):
+ for entry_type, value, _target_id, _main, _category_key in entries:
+ for m in split_index_msg(entry_type, value):
+ catalog.add(m, node)
+
+
+# If set, use the timestamp from SOURCE_DATE_EPOCH
+# https://reproducible-builds.org/specs/source-date-epoch/
if (source_date_epoch := getenv('SOURCE_DATE_EPOCH')) is not None:
timestamp = time.gmtime(float(source_date_epoch))
else:
+ # determine timestamp once to remain unaffected by DST changes during build
timestamp = time.localtime()
ctime = time.strftime('%Y-%m-%d %H:%M%z', timestamp)
-def _is_node_in_substitution_definition(node: nodes.Node) ->bool:
+def should_write(filepath: str, new_content: str) -> bool:
+ if not path.exists(filepath):
+ return True
+ try:
+ with open(filepath, encoding='utf-8') as oldpot:
+ old_content = oldpot.read()
+ old_header_index = old_content.index('"POT-Creation-Date:')
+ new_header_index = new_content.index('"POT-Creation-Date:')
+ old_body_index = old_content.index('"PO-Revision-Date:')
+ new_body_index = new_content.index('"PO-Revision-Date:')
+ return ((old_content[:old_header_index] != new_content[:new_header_index]) or
+ (new_content[new_body_index:] != old_content[old_body_index:]))
+ except ValueError:
+ pass
+
+ return True
+
+
+def _is_node_in_substitution_definition(node: nodes.Node) -> bool:
"""Check "node" to test if it is in a substitution definition."""
- pass
+ while node.parent:
+ if isinstance(node, nodes.substitution_definition):
+ return True
+ node = node.parent
+ return False
class MessageCatalogBuilder(I18nBuilder):
"""
Builds gettext-style message catalogs (.pot files).
"""
+
name = 'gettext'
epilog = __('The message catalogs are in %(outdir)s.')
+
+ def init(self) -> None:
+ super().init()
+ self.create_template_bridge()
+ self.templates.init(self)
+
+ def _collect_templates(self) -> set[str]:
+ template_files = set()
+ for template_path in self.config.templates_path:
+ tmpl_abs_path = path.join(self.app.srcdir, template_path)
+ for dirpath, _dirs, files in walk(tmpl_abs_path):
+ for fn in files:
+ if fn.endswith('.html'):
+ filename = canon_path(path.join(dirpath, fn))
+ template_files.add(filename)
+ return template_files
+
+ def _extract_from_template(self) -> None:
+ files = list(self._collect_templates())
+ files.sort()
+ logger.info(bold(__('building [%s]: ')), self.name, nonl=True)
+ logger.info(__('targets for %d template files'), len(files))
+
+ extract_translations = self.templates.environment.extract_translations
+
+ for template in status_iterator(files, __('reading templates... '), "purple",
+ len(files), self.app.verbosity):
+ try:
+ with open(template, encoding='utf-8') as f:
+ context = f.read()
+ for line, _meth, msg in extract_translations(context):
+ origin = MsgOrigin(template, line)
+ self.catalogs['sphinx'].add(msg, origin)
+ except Exception as exc:
+ msg = f'{template}: {exc!r}'
+ raise ThemeError(msg) from exc
+
+ def build( # type: ignore[misc]
+ self,
+ docnames: Iterable[str] | None,
+ summary: str | None = None,
+ method: Literal['all', 'specific', 'update'] = 'update',
+ ) -> None:
+ self._extract_from_template()
+ super().build(docnames, summary, method)
+
+ def finish(self) -> None:
+ super().finish()
+ context = {
+ 'version': self.config.version,
+ 'copyright': self.config.copyright,
+ 'project': self.config.project,
+ 'last_translator': self.config.gettext_last_translator,
+ 'language_team': self.config.gettext_language_team,
+ 'ctime': ctime,
+ 'display_location': self.config.gettext_location,
+ 'display_uuid': self.config.gettext_uuid,
+ }
+ for textdomain, catalog in status_iterator(self.catalogs.items(),
+ __("writing message catalogs... "),
+ "darkgreen", len(self.catalogs),
+ self.app.verbosity,
+ operator.itemgetter(0)):
+ # noop if config.gettext_compact is set
+ ensuredir(path.join(self.outdir, path.dirname(textdomain)))
+
+ context['messages'] = list(catalog)
+ template_path = [
+ self.app.srcdir / rel_path
+ for rel_path in self.config.templates_path
+ ]
+ renderer = GettextRenderer(template_path, outdir=self.outdir)
+ content = renderer.render('message.pot.jinja', context)
+
+ pofn = path.join(self.outdir, textdomain + '.pot')
+ if should_write(pofn, content):
+ with open(pofn, 'w', encoding='utf-8') as pofile:
+ pofile.write(content)
+
+
+def _gettext_compact_validator(app: Sphinx, config: Config) -> None:
+ gettext_compact = config.gettext_compact
+ # Convert 0/1 from the command line to ``bool`` types
+ if gettext_compact == '0':
+ config.gettext_compact = False
+ elif gettext_compact == '1':
+ config.gettext_compact = True
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(MessageCatalogBuilder)
+
+ app.add_config_value('gettext_compact', True, 'gettext', {bool, str})
+ app.add_config_value('gettext_location', True, 'gettext')
+ app.add_config_value('gettext_uuid', False, 'gettext')
+ app.add_config_value('gettext_auto_build', True, 'env')
+ app.add_config_value('gettext_additional_targets', [], 'env', types={set, list})
+ app.add_config_value('gettext_last_translator', 'FULL NAME <EMAIL@ADDRESS>', 'gettext')
+ app.add_config_value('gettext_language_team', 'LANGUAGE <LL@li.org>', 'gettext')
+ app.connect('config-inited', _gettext_compact_validator, priority=800)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/html/_assets.py b/sphinx/builders/html/_assets.py
index 084435e0e..699a160ee 100644
--- a/sphinx/builders/html/_assets.py
+++ b/sphinx/builders/html/_assets.py
@@ -1,10 +1,13 @@
from __future__ import annotations
+
import os
import warnings
import zlib
from typing import TYPE_CHECKING, Any, NoReturn
+
from sphinx.deprecation import RemovedInSphinx90Warning
from sphinx.errors import ThemeError
+
if TYPE_CHECKING:
from pathlib import Path
@@ -14,53 +17,54 @@ class _CascadingStyleSheet:
priority: int
attributes: dict[str, str]
- def __init__(self, filename: (str | os.PathLike[str]), /, *, priority:
- int=500, rel: str='stylesheet', type: str='text/css', **attributes: str
- ) ->None:
+ def __init__(
+ self,
+ filename: str | os.PathLike[str], /, *,
+ priority: int = 500,
+ rel: str = 'stylesheet',
+ type: str = 'text/css',
+ **attributes: str,
+ ) -> None:
object.__setattr__(self, 'filename', filename)
object.__setattr__(self, 'priority', priority)
- object.__setattr__(self, 'attributes', {'rel': rel, 'type': type} |
- attributes)
+ object.__setattr__(self, 'attributes', {'rel': rel, 'type': type} | attributes)
- def __str__(self) ->str:
+ def __str__(self) -> str:
attr = ', '.join(f'{k}={v!r}' for k, v in self.attributes.items())
- return (
- f'{self.__class__.__name__}({self.filename!r}, priority={self.priority}, {attr})'
- )
+ return (f'{self.__class__.__name__}({self.filename!r}, '
+ f'priority={self.priority}, '
+ f'{attr})')
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if isinstance(other, str):
- warnings.warn(
- 'The str interface for _CascadingStyleSheet objects is deprecated. Use css.filename instead.'
- , RemovedInSphinx90Warning, stacklevel=2)
+ warnings.warn('The str interface for _CascadingStyleSheet objects is deprecated. '
+ 'Use css.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
return self.filename == other
if not isinstance(other, _CascadingStyleSheet):
return NotImplemented
- return (self.filename == other.filename and self.priority == other.
- priority and self.attributes == other.attributes)
+ return (self.filename == other.filename
+ and self.priority == other.priority
+ and self.attributes == other.attributes)
- def __hash__(self) ->int:
- return hash((self.filename, self.priority, *sorted(self.attributes.
- items())))
+ def __hash__(self) -> int:
+ return hash((self.filename, self.priority, *sorted(self.attributes.items())))
- def __setattr__(self, key: str, value: Any) ->NoReturn:
+ def __setattr__(self, key: str, value: Any) -> NoReturn:
msg = f'{self.__class__.__name__} is immutable'
raise AttributeError(msg)
- def __delattr__(self, key: str) ->NoReturn:
+ def __delattr__(self, key: str) -> NoReturn:
msg = f'{self.__class__.__name__} is immutable'
raise AttributeError(msg)
- def __getattr__(self, key: str) ->str:
- warnings.warn(
- 'The str interface for _CascadingStyleSheet objects is deprecated. Use css.filename instead.'
- , RemovedInSphinx90Warning, stacklevel=2)
+ def __getattr__(self, key: str) -> str:
+ warnings.warn('The str interface for _CascadingStyleSheet objects is deprecated. '
+ 'Use css.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
return getattr(os.fspath(self.filename), key)
- def __getitem__(self, key: (int | slice)) ->str:
- warnings.warn(
- 'The str interface for _CascadingStyleSheet objects is deprecated. Use css.filename instead.'
- , RemovedInSphinx90Warning, stacklevel=2)
+ def __getitem__(self, key: int | slice) -> str:
+ warnings.warn('The str interface for _CascadingStyleSheet objects is deprecated. '
+ 'Use css.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
return os.fspath(self.filename)[key]
@@ -69,52 +73,74 @@ class _JavaScript:
priority: int
attributes: dict[str, str]
- def __init__(self, filename: (str | os.PathLike[str]), /, *, priority:
- int=500, **attributes: str) ->None:
+ def __init__(
+ self,
+ filename: str | os.PathLike[str], /, *,
+ priority: int = 500,
+ **attributes: str,
+ ) -> None:
object.__setattr__(self, 'filename', filename)
object.__setattr__(self, 'priority', priority)
object.__setattr__(self, 'attributes', attributes)
- def __str__(self) ->str:
+ def __str__(self) -> str:
attr = ''
if self.attributes:
- attr = ', ' + ', '.join(f'{k}={v!r}' for k, v in self.
- attributes.items())
- return (
- f'{self.__class__.__name__}({self.filename!r}, priority={self.priority}{attr})'
- )
+ attr = ', ' + ', '.join(f'{k}={v!r}' for k, v in self.attributes.items())
+ return (f'{self.__class__.__name__}({self.filename!r}, '
+ f'priority={self.priority}'
+ f'{attr})')
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if isinstance(other, str):
- warnings.warn(
- 'The str interface for _JavaScript objects is deprecated. Use js.filename instead.'
- , RemovedInSphinx90Warning, stacklevel=2)
+ warnings.warn('The str interface for _JavaScript objects is deprecated. '
+ 'Use js.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
return self.filename == other
if not isinstance(other, _JavaScript):
return NotImplemented
- return (self.filename == other.filename and self.priority == other.
- priority and self.attributes == other.attributes)
+ return (self.filename == other.filename
+ and self.priority == other.priority
+ and self.attributes == other.attributes)
- def __hash__(self) ->int:
- return hash((self.filename, self.priority, *sorted(self.attributes.
- items())))
+ def __hash__(self) -> int:
+ return hash((self.filename, self.priority, *sorted(self.attributes.items())))
- def __setattr__(self, key: str, value: Any) ->NoReturn:
+ def __setattr__(self, key: str, value: Any) -> NoReturn:
msg = f'{self.__class__.__name__} is immutable'
raise AttributeError(msg)
- def __delattr__(self, key: str) ->NoReturn:
+ def __delattr__(self, key: str) -> NoReturn:
msg = f'{self.__class__.__name__} is immutable'
raise AttributeError(msg)
- def __getattr__(self, key: str) ->str:
- warnings.warn(
- 'The str interface for _JavaScript objects is deprecated. Use js.filename instead.'
- , RemovedInSphinx90Warning, stacklevel=2)
+ def __getattr__(self, key: str) -> str:
+ warnings.warn('The str interface for _JavaScript objects is deprecated. '
+ 'Use js.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
return getattr(os.fspath(self.filename), key)
- def __getitem__(self, key: (int | slice)) ->str:
- warnings.warn(
- 'The str interface for _JavaScript objects is deprecated. Use js.filename instead.'
- , RemovedInSphinx90Warning, stacklevel=2)
+ def __getitem__(self, key: int | slice) -> str:
+ warnings.warn('The str interface for _JavaScript objects is deprecated. '
+ 'Use js.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
return os.fspath(self.filename)[key]
+
+
+def _file_checksum(outdir: Path, filename: str | os.PathLike[str]) -> str:
+ filename = os.fspath(filename)
+ # Don't generate checksums for HTTP URIs
+ if '://' in filename:
+ return ''
+ # Some themes and extensions have used query strings
+ # for a similar asset checksum feature.
+ # As we cannot safely strip the query string,
+ # raise an error to the user.
+ if '?' in filename:
+ msg = f'Local asset file paths must not contain query strings: {filename!r}'
+ raise ThemeError(msg)
+ try:
+ # Remove all carriage returns to avoid checksum differences
+ content = outdir.joinpath(filename).read_bytes().translate(None, b'\r')
+ except FileNotFoundError:
+ return ''
+ if not content:
+ return ''
+ return f'{zlib.crc32(content):08x}'
diff --git a/sphinx/builders/html/_build_info.py b/sphinx/builders/html/_build_info.py
index 124c16e83..5b364c0d9 100644
--- a/sphinx/builders/html/_build_info.py
+++ b/sphinx/builders/html/_build_info.py
@@ -1,13 +1,18 @@
"""Record metadata for the build process."""
+
from __future__ import annotations
+
import hashlib
import types
from typing import TYPE_CHECKING
+
from sphinx.locale import __
+
if TYPE_CHECKING:
from collections.abc import Set
from pathlib import Path
from typing import Any
+
from sphinx.config import Config, _ConfigRebuild
from sphinx.util.tags import Tags
@@ -19,26 +24,71 @@ class BuildInfo:
This class is a manipulator for the file.
"""
- def __init__(self, config: (Config | None)=None, tags: (Tags | None)=
- None, config_categories: Set[_ConfigRebuild]=frozenset()) ->None:
+ @classmethod
+ def load(cls: type[BuildInfo], filename: Path, /) -> BuildInfo:
+ content = filename.read_text(encoding="utf-8")
+ lines = content.splitlines()
+
+ version = lines[0].rstrip()
+ if version != '# Sphinx build info version 1':
+ msg = __('failed to read broken build info file (unknown version)')
+ raise ValueError(msg)
+
+ if not lines[2].startswith('config: '):
+ msg = __('failed to read broken build info file (missing config entry)')
+ raise ValueError(msg)
+ if not lines[3].startswith('tags: '):
+ msg = __('failed to read broken build info file (missing tags entry)')
+ raise ValueError(msg)
+
+ build_info = BuildInfo()
+ build_info.config_hash = lines[2].removeprefix('config: ').strip()
+ build_info.tags_hash = lines[3].removeprefix('tags: ').strip()
+ return build_info
+
+ def __init__(
+ self,
+ config: Config | None = None,
+ tags: Tags | None = None,
+ config_categories: Set[_ConfigRebuild] = frozenset(),
+ ) -> None:
self.config_hash = ''
self.tags_hash = ''
+
if config:
- values = {c.name: c.value for c in config.filter(config_categories)
- }
+ values = {c.name: c.value for c in config.filter(config_categories)}
self.config_hash = _stable_hash(values)
+
if tags:
self.tags_hash = _stable_hash(sorted(tags))
- def __eq__(self, other: BuildInfo) ->bool:
- return (self.config_hash == other.config_hash and self.tags_hash ==
- other.tags_hash)
+ def __eq__(self, other: BuildInfo) -> bool: # type: ignore[override]
+ return (self.config_hash == other.config_hash and
+ self.tags_hash == other.tags_hash)
+
+ def dump(self, filename: Path, /) -> None:
+ build_info = (
+ '# Sphinx build info version 1\n'
+ '# This file records the configuration used when building these files. '
+ 'When it is not found, a full rebuild will be done.\n'
+ f'config: {self.config_hash}\n'
+ f'tags: {self.tags_hash}\n'
+ )
+ filename.write_text(build_info, encoding="utf-8")
-def _stable_hash(obj: Any) ->str:
+def _stable_hash(obj: Any) -> str:
"""Return a stable hash for a Python data structure.
We can't just use the md5 of str(obj) as the order of collections
may be random.
"""
- pass
+ if isinstance(obj, dict):
+ obj = sorted(map(_stable_hash, obj.items()))
+ if isinstance(obj, list | tuple | set | frozenset):
+ obj = sorted(map(_stable_hash, obj))
+ elif isinstance(obj, type | types.FunctionType):
+ # The default repr() of functions includes the ID, which is not ideal.
+ # We use the fully qualified name instead.
+ obj = f'{obj.__module__}.{obj.__qualname__}'
+ return hashlib.md5(str(obj).encode(), usedforsecurity=False).hexdigest()
diff --git a/sphinx/builders/html/transforms.py b/sphinx/builders/html/transforms.py
index 261a2a96d..a36588c7b 100644
--- a/sphinx/builders/html/transforms.py
+++ b/sphinx/builders/html/transforms.py
@@ -1,10 +1,15 @@
"""Transforms for HTML builder."""
+
from __future__ import annotations
+
import re
from typing import TYPE_CHECKING, Any
+
from docutils import nodes
+
from sphinx.transforms.post_transforms import SphinxPostTransform
from sphinx.util.nodes import NodeMatcher
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
@@ -27,8 +32,57 @@ class KeyboardTransform(SphinxPostTransform):
<literal class="kbd">
x
"""
+
default_priority = 400
- formats = 'html',
- pattern = re.compile('(?<=.)(-|\\+|\\^|\\s+)(?=.)')
- multiwords_keys = ('caps', 'lock'), ('page', 'down'), ('page', 'up'), (
- 'scroll', 'lock'), ('num', 'lock'), ('sys', 'rq'), ('back', 'space')
+ formats = ('html',)
+ pattern = re.compile(r'(?<=.)(-|\+|\^|\s+)(?=.)')
+ multiwords_keys = (('caps', 'lock'),
+ ('page', 'down'),
+ ('page', 'up'),
+ ('scroll', 'lock'),
+ ('num', 'lock'),
+ ('sys', 'rq'),
+ ('back', 'space'))
+
+ def run(self, **kwargs: Any) -> None:
+ matcher = NodeMatcher(nodes.literal, classes=["kbd"])
+ # this list must be pre-created as during iteration new nodes
+ # are added which match the condition in the NodeMatcher.
+ for node in list(matcher.findall(self.document)):
+ parts = self.pattern.split(node[-1].astext())
+ if len(parts) == 1 or self.is_multiwords_key(parts):
+ continue
+
+ node['classes'].append('compound')
+ node.pop()
+ while parts:
+ if self.is_multiwords_key(parts):
+ key = ''.join(parts[:3])
+ parts[:3] = []
+ else:
+ key = parts.pop(0)
+ node += nodes.literal('', key, classes=["kbd"])
+
+ try:
+ # key separator (ex. -, +, ^)
+ sep = parts.pop(0)
+ node += nodes.Text(sep)
+ except IndexError:
+ pass
+
+ def is_multiwords_key(self, parts: list[str]) -> bool:
+ if len(parts) >= 3 and parts[1].strip() == '':
+ name = parts[0].lower(), parts[2].lower()
+ return name in self.multiwords_keys
+ else:
+ return False
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_post_transform(KeyboardTransform)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/latex/constants.py b/sphinx/builders/latex/constants.py
index 7d8f9fe64..9da66e826 100644
--- a/sphinx/builders/latex/constants.py
+++ b/sphinx/builders/latex/constants.py
@@ -1,140 +1,215 @@
"""constants for LaTeX builder."""
+
from __future__ import annotations
+
from typing import Any
-PDFLATEX_DEFAULT_FONTPKG = """
-\\usepackage{tgtermes}
-\\usepackage{tgheros}
-\\renewcommand{\\ttdefault}{txtt}
-"""
-PDFLATEX_DEFAULT_FONTSUBSTITUTION = """
-\\expandafter\\ifx\\csname T@LGR\\endcsname\\relax
-\\else
+
+PDFLATEX_DEFAULT_FONTPKG = r'''
+\usepackage{tgtermes}
+\usepackage{tgheros}
+\renewcommand{\ttdefault}{txtt}
+'''
+
+PDFLATEX_DEFAULT_FONTSUBSTITUTION = r'''
+\expandafter\ifx\csname T@LGR\endcsname\relax
+\else
% LGR was declared as font encoding
- \\substitutefont{LGR}{\\rmdefault}{cmr}
- \\substitutefont{LGR}{\\sfdefault}{cmss}
- \\substitutefont{LGR}{\\ttdefault}{cmtt}
-\\fi
-\\expandafter\\ifx\\csname T@X2\\endcsname\\relax
- \\expandafter\\ifx\\csname T@T2A\\endcsname\\relax
- \\else
+ \substitutefont{LGR}{\rmdefault}{cmr}
+ \substitutefont{LGR}{\sfdefault}{cmss}
+ \substitutefont{LGR}{\ttdefault}{cmtt}
+\fi
+\expandafter\ifx\csname T@X2\endcsname\relax
+ \expandafter\ifx\csname T@T2A\endcsname\relax
+ \else
% T2A was declared as font encoding
- \\substitutefont{T2A}{\\rmdefault}{cmr}
- \\substitutefont{T2A}{\\sfdefault}{cmss}
- \\substitutefont{T2A}{\\ttdefault}{cmtt}
- \\fi
-\\else
+ \substitutefont{T2A}{\rmdefault}{cmr}
+ \substitutefont{T2A}{\sfdefault}{cmss}
+ \substitutefont{T2A}{\ttdefault}{cmtt}
+ \fi
+\else
% X2 was declared as font encoding
- \\substitutefont{X2}{\\rmdefault}{cmr}
- \\substitutefont{X2}{\\sfdefault}{cmss}
- \\substitutefont{X2}{\\ttdefault}{cmtt}
-\\fi
-"""
-XELATEX_DEFAULT_FONTPKG = """
-\\setmainfont{FreeSerif}[
+ \substitutefont{X2}{\rmdefault}{cmr}
+ \substitutefont{X2}{\sfdefault}{cmss}
+ \substitutefont{X2}{\ttdefault}{cmtt}
+\fi
+'''
+
+XELATEX_DEFAULT_FONTPKG = r'''
+\setmainfont{FreeSerif}[
Extension = .otf,
UprightFont = *,
ItalicFont = *Italic,
BoldFont = *Bold,
BoldItalicFont = *BoldItalic
]
-\\setsansfont{FreeSans}[
+\setsansfont{FreeSans}[
Extension = .otf,
UprightFont = *,
ItalicFont = *Oblique,
BoldFont = *Bold,
BoldItalicFont = *BoldOblique,
]
-\\setmonofont{FreeMono}[
+\setmonofont{FreeMono}[
Extension = .otf,
UprightFont = *,
ItalicFont = *Oblique,
BoldFont = *Bold,
BoldItalicFont = *BoldOblique,
]
-"""
-XELATEX_GREEK_DEFAULT_FONTPKG = XELATEX_DEFAULT_FONTPKG + """
-\\newfontfamily\\greekfont{FreeSerif}""" + """
-\\newfontfamily\\greekfontsf{FreeSans}""" + """
-\\newfontfamily\\greekfonttt{FreeMono}"""
+'''
+
+XELATEX_GREEK_DEFAULT_FONTPKG = (XELATEX_DEFAULT_FONTPKG +
+ '\n\\newfontfamily\\greekfont{FreeSerif}' +
+ '\n\\newfontfamily\\greekfontsf{FreeSans}' +
+ '\n\\newfontfamily\\greekfonttt{FreeMono}')
+
LUALATEX_DEFAULT_FONTPKG = XELATEX_DEFAULT_FONTPKG
-DEFAULT_SETTINGS: dict[str, Any] = {'latex_engine': 'pdflatex', 'papersize':
- '', 'pointsize': '', 'pxunit': '.75bp', 'classoptions': '',
- 'extraclassoptions': '', 'maxlistdepth': '', 'sphinxpkgoptions': '',
- 'sphinxsetup': '', 'fvset': '\\fvset{fontsize=auto}',
- 'passoptionstopackages': '', 'geometry': '\\usepackage{geometry}',
- 'inputenc': '', 'utf8extra': '', 'cmappkg': '\\usepackage{cmap}',
- 'fontenc': '\\usepackage[T1]{fontenc}', 'amsmath':
- '\\usepackage{amsmath,amssymb,amstext}', 'multilingual': '', 'babel':
- '\\usepackage{babel}', 'polyglossia': '', 'fontpkg':
- PDFLATEX_DEFAULT_FONTPKG, 'fontsubstitution':
- PDFLATEX_DEFAULT_FONTSUBSTITUTION, 'substitutefont': '', 'textcyrillic':
- '', 'textgreek': '\\usepackage{textalpha}', 'fncychap':
- '\\usepackage[Bjarne]{fncychap}', 'hyperref':
- """% Include hyperref last.
-\\usepackage{hyperref}
-% Fix anchor placement for figures with captions.
-\\usepackage{hypcap}% it must be loaded after hyperref.
-% Set up styles of URL: it should be placed after hyperref.
-\\urlstyle{same}"""
- , 'contentsname': '', 'extrapackages': '', 'preamble': '', 'title': '',
- 'release': '', 'author': '', 'releasename': '', 'makeindex':
- '\\makeindex', 'shorthandoff': '', 'maketitle': '\\sphinxmaketitle',
- 'tableofcontents': '\\sphinxtableofcontents', 'atendofbody': '',
- 'printindex': '\\printindex', 'transition':
- """
-\\bigskip\\hrule\\bigskip
+DEFAULT_SETTINGS: dict[str, Any] = {
+ 'latex_engine': 'pdflatex',
+ 'papersize': '',
+ 'pointsize': '',
+ 'pxunit': '.75bp',
+ 'classoptions': '',
+ 'extraclassoptions': '',
+ 'maxlistdepth': '',
+ 'sphinxpkgoptions': '',
+ 'sphinxsetup': '',
+ 'fvset': '\\fvset{fontsize=auto}',
+ 'passoptionstopackages': '',
+ 'geometry': '\\usepackage{geometry}',
+ 'inputenc': '',
+ 'utf8extra': '',
+ 'cmappkg': '\\usepackage{cmap}',
+ 'fontenc': '\\usepackage[T1]{fontenc}',
+ 'amsmath': '\\usepackage{amsmath,amssymb,amstext}',
+ 'multilingual': '',
+ 'babel': '\\usepackage{babel}',
+ 'polyglossia': '',
+ 'fontpkg': PDFLATEX_DEFAULT_FONTPKG,
+ 'fontsubstitution': PDFLATEX_DEFAULT_FONTSUBSTITUTION,
+ 'substitutefont': '',
+ 'textcyrillic': '',
+ 'textgreek': '\\usepackage{textalpha}',
+ 'fncychap': '\\usepackage[Bjarne]{fncychap}',
+ 'hyperref': ('% Include hyperref last.\n'
+ '\\usepackage{hyperref}\n'
+ '% Fix anchor placement for figures with captions.\n'
+ '\\usepackage{hypcap}% it must be loaded after hyperref.\n'
+ '% Set up styles of URL: it should be placed after hyperref.\n'
+ '\\urlstyle{same}'),
+ 'contentsname': '',
+ 'extrapackages': '',
+ 'preamble': '',
+ 'title': '',
+ 'release': '',
+ 'author': '',
+ 'releasename': '',
+ 'makeindex': '\\makeindex',
+ 'shorthandoff': '',
+ 'maketitle': '\\sphinxmaketitle',
+ 'tableofcontents': '\\sphinxtableofcontents',
+ 'atendofbody': '',
+ 'printindex': '\\printindex',
+ 'transition': '\n\n\\bigskip\\hrule\\bigskip\n\n',
+ 'figure_align': 'htbp',
+ 'tocdepth': '',
+ 'secnumdepth': '',
+}
+
+ADDITIONAL_SETTINGS: dict[Any, dict[str, Any]] = {
+ 'pdflatex': {
+ 'inputenc': '\\usepackage[utf8]{inputenc}',
+ 'utf8extra': ('\\ifdefined\\DeclareUnicodeCharacter\n'
+ '% support both utf8 and utf8x syntaxes\n'
+ ' \\ifdefined\\DeclareUnicodeCharacterAsOptional\n'
+ ' \\def\\sphinxDUC#1{\\DeclareUnicodeCharacter{"#1}}\n'
+ ' \\else\n'
+ ' \\let\\sphinxDUC\\DeclareUnicodeCharacter\n'
+ ' \\fi\n'
+ ' \\sphinxDUC{00A0}{\\nobreakspace}\n'
+ ' \\sphinxDUC{2500}{\\sphinxunichar{2500}}\n'
+ ' \\sphinxDUC{2502}{\\sphinxunichar{2502}}\n'
+ ' \\sphinxDUC{2514}{\\sphinxunichar{2514}}\n'
+ ' \\sphinxDUC{251C}{\\sphinxunichar{251C}}\n'
+ ' \\sphinxDUC{2572}{\\textbackslash}\n'
+ '\\fi'),
+ },
+ 'xelatex': {
+ 'latex_engine': 'xelatex',
+ 'polyglossia': '\\usepackage{polyglossia}',
+ 'babel': '',
+ 'fontenc': ('\\usepackage{fontspec}\n'
+ '\\defaultfontfeatures[\\rmfamily,\\sffamily,\\ttfamily]{}'),
+ 'fontpkg': XELATEX_DEFAULT_FONTPKG,
+ 'fvset': '\\fvset{fontsize=\\small}',
+ 'fontsubstitution': '',
+ 'textgreek': '',
+ 'utf8extra': ('\\catcode`^^^^00a0\\active\\protected\\def^^^^00a0'
+ '{\\leavevmode\\nobreak\\ }'),
+ },
+ 'lualatex': {
+ 'latex_engine': 'lualatex',
+ 'polyglossia': '\\usepackage{polyglossia}',
+ 'babel': '',
+ 'fontenc': ('\\usepackage{fontspec}\n'
+ '\\defaultfontfeatures[\\rmfamily,\\sffamily,\\ttfamily]{}'),
+ 'fontpkg': LUALATEX_DEFAULT_FONTPKG,
+ 'fvset': '\\fvset{fontsize=\\small}',
+ 'fontsubstitution': '',
+ 'textgreek': '',
+ 'utf8extra': ('\\catcode`^^^^00a0\\active\\protected\\def^^^^00a0'
+ '{\\leavevmode\\nobreak\\ }'),
+ },
+ 'platex': {
+ 'latex_engine': 'platex',
+ 'babel': '',
+ 'classoptions': ',dvipdfmx',
+ 'fontpkg': PDFLATEX_DEFAULT_FONTPKG,
+ 'fontsubstitution': '',
+ 'textgreek': '',
+ 'fncychap': '',
+ 'geometry': '\\usepackage[dvipdfm]{geometry}',
+ },
+ 'uplatex': {
+ 'latex_engine': 'uplatex',
+ 'babel': '',
+ 'classoptions': ',dvipdfmx',
+ 'fontpkg': PDFLATEX_DEFAULT_FONTPKG,
+ 'fontsubstitution': '',
+ 'textgreek': '',
+ 'fncychap': '',
+ 'geometry': '\\usepackage[dvipdfm]{geometry}',
+ },
+
+ # special settings for latex_engine + language_code
+ ('lualatex', 'fr'): {
+ # use babel instead of polyglossia by default
+ 'polyglossia': '',
+ 'babel': '\\usepackage{babel}',
+ },
+ ('xelatex', 'fr'): {
+ # use babel instead of polyglossia by default
+ 'polyglossia': '',
+ 'babel': '\\usepackage{babel}',
+ },
+ ('xelatex', 'zh'): {
+ 'polyglossia': '',
+ 'babel': '\\usepackage{babel}',
+ 'fontenc': '\\usepackage{xeCJK}',
+ # set formatcom=\xeCJKVerbAddon to prevent xeCJK from adding extra spaces in
+ # fancyvrb Verbatim environment.
+ 'fvset': '\\fvset{fontsize=\\small,formatcom=\\xeCJKVerbAddon}',
+ },
+ ('xelatex', 'el'): {
+ 'fontpkg': XELATEX_GREEK_DEFAULT_FONTPKG,
+ },
+}
+
-""", 'figure_align': 'htbp', 'tocdepth':
- '', 'secnumdepth': ''}
-ADDITIONAL_SETTINGS: dict[Any, dict[str, Any]] = {'pdflatex': {'inputenc':
- '\\usepackage[utf8]{inputenc}', 'utf8extra':
- """\\ifdefined\\DeclareUnicodeCharacter
-% support both utf8 and utf8x syntaxes
- \\ifdefined\\DeclareUnicodeCharacterAsOptional
- \\def\\sphinxDUC#1{\\DeclareUnicodeCharacter{"#1}}
- \\else
- \\let\\sphinxDUC\\DeclareUnicodeCharacter
- \\fi
- \\sphinxDUC{00A0}{\\nobreakspace}
- \\sphinxDUC{2500}{\\sphinxunichar{2500}}
- \\sphinxDUC{2502}{\\sphinxunichar{2502}}
- \\sphinxDUC{2514}{\\sphinxunichar{2514}}
- \\sphinxDUC{251C}{\\sphinxunichar{251C}}
- \\sphinxDUC{2572}{\\textbackslash}
-\\fi"""
- }, 'xelatex': {'latex_engine': 'xelatex', 'polyglossia':
- '\\usepackage{polyglossia}', 'babel': '', 'fontenc':
- """\\usepackage{fontspec}
-\\defaultfontfeatures[\\rmfamily,\\sffamily,\\ttfamily]{}"""
- , 'fontpkg': XELATEX_DEFAULT_FONTPKG, 'fvset':
- '\\fvset{fontsize=\\small}', 'fontsubstitution': '', 'textgreek': '',
- 'utf8extra':
- '\\catcode`^^^^00a0\\active\\protected\\def^^^^00a0{\\leavevmode\\nobreak\\ }'
- }, 'lualatex': {'latex_engine': 'lualatex', 'polyglossia':
- '\\usepackage{polyglossia}', 'babel': '', 'fontenc':
- """\\usepackage{fontspec}
-\\defaultfontfeatures[\\rmfamily,\\sffamily,\\ttfamily]{}"""
- , 'fontpkg': LUALATEX_DEFAULT_FONTPKG, 'fvset':
- '\\fvset{fontsize=\\small}', 'fontsubstitution': '', 'textgreek': '',
- 'utf8extra':
- '\\catcode`^^^^00a0\\active\\protected\\def^^^^00a0{\\leavevmode\\nobreak\\ }'
- }, 'platex': {'latex_engine': 'platex', 'babel': '', 'classoptions':
- ',dvipdfmx', 'fontpkg': PDFLATEX_DEFAULT_FONTPKG, 'fontsubstitution':
- '', 'textgreek': '', 'fncychap': '', 'geometry':
- '\\usepackage[dvipdfm]{geometry}'}, 'uplatex': {'latex_engine':
- 'uplatex', 'babel': '', 'classoptions': ',dvipdfmx', 'fontpkg':
- PDFLATEX_DEFAULT_FONTPKG, 'fontsubstitution': '', 'textgreek': '',
- 'fncychap': '', 'geometry': '\\usepackage[dvipdfm]{geometry}'}, (
- 'lualatex', 'fr'): {'polyglossia': '', 'babel': '\\usepackage{babel}'},
- ('xelatex', 'fr'): {'polyglossia': '', 'babel': '\\usepackage{babel}'},
- ('xelatex', 'zh'): {'polyglossia': '', 'babel': '\\usepackage{babel}',
- 'fontenc': '\\usepackage{xeCJK}', 'fvset':
- '\\fvset{fontsize=\\small,formatcom=\\xeCJKVerbAddon}'}, ('xelatex',
- 'el'): {'fontpkg': XELATEX_GREEK_DEFAULT_FONTPKG}}
-SHORTHANDOFF = """
-\\ifdefined\\shorthandoff
- \\ifnum\\catcode`\\=\\string=\\active\\shorthandoff{=}\\fi
- \\ifnum\\catcode`\\"=\\active\\shorthandoff{"}\\fi
-\\fi
-"""
+SHORTHANDOFF = r'''
+\ifdefined\shorthandoff
+ \ifnum\catcode`\=\string=\active\shorthandoff{=}\fi
+ \ifnum\catcode`\"=\active\shorthandoff{"}\fi
+\fi
+'''
diff --git a/sphinx/builders/latex/nodes.py b/sphinx/builders/latex/nodes.py
index cdf9949a3..68b743d40 100644
--- a/sphinx/builders/latex/nodes.py
+++ b/sphinx/builders/latex/nodes.py
@@ -1,31 +1,41 @@
"""Additional nodes for LaTeX writer."""
+
from docutils import nodes
class captioned_literal_block(nodes.container):
"""A node for a container of literal_block having a caption."""
+
pass
class footnotemark(nodes.Inline, nodes.Referential, nodes.TextElement):
- """A node represents ``\\footnotemark``."""
+ r"""A node represents ``\footnotemark``."""
+
pass
-class footnotetext(nodes.General, nodes.BackLinkable, nodes.Element, nodes.
- Labeled, nodes.Targetable):
- """A node represents ``\\footnotetext``."""
+class footnotetext(nodes.General, nodes.BackLinkable, nodes.Element,
+ nodes.Labeled, nodes.Targetable):
+ r"""A node represents ``\footnotetext``."""
class math_reference(nodes.Inline, nodes.Referential, nodes.TextElement):
"""A node for a reference for equation."""
+
pass
class thebibliography(nodes.container):
"""A node for wrapping bibliographies."""
+
pass
-HYPERLINK_SUPPORT_NODES = (nodes.figure, nodes.literal_block, nodes.table,
- nodes.section, captioned_literal_block)
+HYPERLINK_SUPPORT_NODES = (
+ nodes.figure,
+ nodes.literal_block,
+ nodes.table,
+ nodes.section,
+ captioned_literal_block,
+)
diff --git a/sphinx/builders/latex/theming.py b/sphinx/builders/latex/theming.py
index 671dc8ff2..21b49e8ab 100644
--- a/sphinx/builders/latex/theming.py
+++ b/sphinx/builders/latex/theming.py
@@ -1,23 +1,29 @@
"""Theming support for LaTeX builder."""
+
from __future__ import annotations
+
import configparser
from os import path
from typing import TYPE_CHECKING
+
from sphinx.errors import ThemeError
from sphinx.locale import __
from sphinx.util import logging
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.config import Config
+
logger = logging.getLogger(__name__)
class Theme:
"""A set of LaTeX configurations."""
+
LATEX_ELEMENTS_KEYS = ['papersize', 'pointsize']
UPDATABLE_KEYS = ['papersize', 'pointsize']
- def __init__(self, name: str) ->None:
+ def __init__(self, name: str) -> None:
self.name = name
self.docclass = name
self.wrapperclass = name
@@ -25,24 +31,37 @@ class Theme:
self.pointsize = '10pt'
self.toplevel_sectioning = 'chapter'
- def update(self, config: Config) ->None:
+ def update(self, config: Config) -> None:
"""Override theme settings by user's configuration."""
- pass
+ for key in self.LATEX_ELEMENTS_KEYS:
+ if config.latex_elements.get(key):
+ value = config.latex_elements[key]
+ setattr(self, key, value)
+
+ for key in self.UPDATABLE_KEYS:
+ if key in config.latex_theme_options:
+ value = config.latex_theme_options[key]
+ setattr(self, key, value)
class BuiltInTheme(Theme):
"""A built-in LaTeX theme."""
- def __init__(self, name: str, config: Config) ->None:
+ def __init__(self, name: str, config: Config) -> None:
super().__init__(name)
+
if name == 'howto':
self.docclass = config.latex_docclass.get('howto', 'article')
else:
self.docclass = config.latex_docclass.get('manual', 'report')
+
if name in ('manual', 'howto'):
self.wrapperclass = 'sphinx' + name
else:
self.wrapperclass = name
+
+ # we assume LaTeX class provides \chapter command except in case
+ # of non-Japanese 'howto' case
if name == 'howto' and not self.docclass.startswith('j'):
self.toplevel_sectioning = 'section'
else:
@@ -51,23 +70,26 @@ class BuiltInTheme(Theme):
class UserTheme(Theme):
"""A user defined LaTeX theme."""
+
REQUIRED_CONFIG_KEYS = ['docclass', 'wrapperclass']
OPTIONAL_CONFIG_KEYS = ['papersize', 'pointsize', 'toplevel_sectioning']
- def __init__(self, name: str, filename: str) ->None:
+ def __init__(self, name: str, filename: str) -> None:
super().__init__(name)
self.config = configparser.RawConfigParser()
self.config.read(path.join(filename), encoding='utf-8')
+
for key in self.REQUIRED_CONFIG_KEYS:
try:
value = self.config.get('theme', key)
setattr(self, key, value)
except configparser.NoSectionError as exc:
raise ThemeError(__('%r doesn\'t have "theme" setting') %
- filename) from exc
+ filename) from exc
except configparser.NoOptionError as exc:
- raise ThemeError(__('%r doesn\'t have "%s" setting') % (
- filename, exc.args[0])) from exc
+ raise ThemeError(__('%r doesn\'t have "%s" setting') %
+ (filename, exc.args[0])) from exc
+
for key in self.OPTIONAL_CONFIG_KEYS:
try:
value = self.config.get('theme', key)
@@ -79,21 +101,35 @@ class UserTheme(Theme):
class ThemeFactory:
"""A factory class for LaTeX Themes."""
- def __init__(self, app: Sphinx) ->None:
+ def __init__(self, app: Sphinx) -> None:
self.themes: dict[str, Theme] = {}
- self.theme_paths = [path.join(app.srcdir, p) for p in app.config.
- latex_theme_path]
+ self.theme_paths = [path.join(app.srcdir, p) for p in app.config.latex_theme_path]
self.config = app.config
self.load_builtin_themes(app.config)
- def load_builtin_themes(self, config: Config) ->None:
+ def load_builtin_themes(self, config: Config) -> None:
"""Load built-in themes."""
- pass
+ self.themes['manual'] = BuiltInTheme('manual', config)
+ self.themes['howto'] = BuiltInTheme('howto', config)
- def get(self, name: str) ->Theme:
+ def get(self, name: str) -> Theme:
"""Get a theme for given *name*."""
- pass
+ if name in self.themes:
+ theme = self.themes[name]
+ else:
+ theme = self.find_user_theme(name) or Theme(name)
+
+ theme.update(self.config)
+ return theme
- def find_user_theme(self, name: str) ->(Theme | None):
+ def find_user_theme(self, name: str) -> Theme | None:
"""Find a theme named as *name* from latex_theme_path."""
- pass
+ for theme_path in self.theme_paths:
+ config_path = path.join(theme_path, name, 'theme.conf')
+ if path.isfile(config_path):
+ try:
+ return UserTheme(name, config_path)
+ except ThemeError as exc:
+ logger.warning(exc)
+
+ return None
diff --git a/sphinx/builders/latex/transforms.py b/sphinx/builders/latex/transforms.py
index 434115b93..c85dce530 100644
--- a/sphinx/builders/latex/transforms.py
+++ b/sphinx/builders/latex/transforms.py
@@ -1,32 +1,57 @@
"""Transforms for LaTeX builder."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes
from docutils.transforms.references import Substitutions
+
from sphinx import addnodes
-from sphinx.builders.latex.nodes import captioned_literal_block, footnotemark, footnotetext, math_reference, thebibliography
+from sphinx.builders.latex.nodes import (
+ captioned_literal_block,
+ footnotemark,
+ footnotetext,
+ math_reference,
+ thebibliography,
+)
from sphinx.domains.citation import CitationDomain
from sphinx.locale import __
from sphinx.transforms import SphinxTransform
from sphinx.transforms.post_transforms import SphinxPostTransform
from sphinx.util.nodes import NodeMatcher
+
if TYPE_CHECKING:
from docutils.nodes import Element, Node
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
-URI_SCHEMES = 'mailto:', 'http:', 'https:', 'ftp:'
+
+URI_SCHEMES = ('mailto:', 'http:', 'https:', 'ftp:')
class FootnoteDocnameUpdater(SphinxTransform):
"""Add docname to footnote and footnote_reference nodes."""
+
default_priority = 700
- TARGET_NODES = nodes.footnote, nodes.footnote_reference
+ TARGET_NODES = (nodes.footnote, nodes.footnote_reference)
+
+ def apply(self, **kwargs: Any) -> None:
+ matcher = NodeMatcher(*self.TARGET_NODES)
+ for node in matcher.findall(self.document):
+ node['docname'] = self.env.docname
class SubstitutionDefinitionsRemover(SphinxPostTransform):
"""Remove ``substitution_definition`` nodes from doctrees."""
+
+ # should be invoked after Substitutions process
default_priority = Substitutions.default_priority + 1
- formats = 'latex',
+ formats = ('latex',)
+
+ def run(self, **kwargs: Any) -> None:
+ for node in list(self.document.findall(nodes.substitution_definition)):
+ node.parent.remove(node)
class ShowUrlsTransform(SphinxPostTransform):
@@ -36,20 +61,136 @@ class ShowUrlsTransform(SphinxPostTransform):
.. note:: This transform is used for integrated doctree
"""
+
default_priority = 400
- formats = 'latex',
+ formats = ('latex',)
+
+ # references are expanded to footnotes (or not)
expanded = False
+ def run(self, **kwargs: Any) -> None:
+ try:
+ # replace id_prefix temporarily
+ settings: Any = self.document.settings
+ id_prefix = settings.id_prefix
+ settings.id_prefix = 'show_urls'
+
+ self.expand_show_urls()
+ if self.expanded:
+ self.renumber_footnotes()
+ finally:
+ # restore id_prefix
+ settings.id_prefix = id_prefix
+
+ def expand_show_urls(self) -> None:
+ show_urls = self.config.latex_show_urls
+ if show_urls is False or show_urls == 'no':
+ return
+
+ for node in list(self.document.findall(nodes.reference)):
+ uri = node.get('refuri', '')
+ if uri.startswith(URI_SCHEMES):
+ if uri.startswith('mailto:'):
+ uri = uri[7:]
+ if node.astext() != uri:
+ index = node.parent.index(node)
+ docname = self.get_docname_for_node(node)
+ if show_urls == 'footnote':
+ fn, fnref = self.create_footnote(uri, docname)
+ node.parent.insert(index + 1, fn)
+ node.parent.insert(index + 2, fnref)
+
+ self.expanded = True
+ else: # all other true values (b/w compat)
+ textnode = nodes.Text(" (%s)" % uri)
+ node.parent.insert(index + 1, textnode)
+
+ def get_docname_for_node(self, node: Node) -> str:
+ while node:
+ if isinstance(node, nodes.document):
+ return self.env.path2doc(node['source']) or ''
+ elif isinstance(node, addnodes.start_of_file):
+ return node['docname']
+ else:
+ node = node.parent
+
+ try:
+ source = node['source']
+ except TypeError:
+ raise ValueError(__('Failed to get a docname!')) from None
+ raise ValueError(__('Failed to get a docname '
+ 'for source {source!r}!').format(source=source))
+
+ def create_footnote(
+ self, uri: str, docname: str,
+ ) -> tuple[nodes.footnote, nodes.footnote_reference]:
+ reference = nodes.reference('', nodes.Text(uri), refuri=uri, nolinkurl=True)
+ footnote = nodes.footnote(uri, auto=1, docname=docname)
+ footnote['names'].append('#')
+ footnote += nodes.label('', '#')
+ footnote += nodes.paragraph('', '', reference)
+ self.document.note_autofootnote(footnote)
+
+ footnote_ref = nodes.footnote_reference('[#]_', auto=1,
+ refid=footnote['ids'][0], docname=docname)
+ footnote_ref += nodes.Text('#')
+ self.document.note_autofootnote_ref(footnote_ref)
+ footnote.add_backref(footnote_ref['ids'][0])
+
+ return footnote, footnote_ref
+
+ def renumber_footnotes(self) -> None:
+ collector = FootnoteCollector(self.document)
+ self.document.walkabout(collector)
+
+ num = 0
+ for footnote in collector.auto_footnotes:
+ # search unused footnote number
+ while True:
+ num += 1
+ if str(num) not in collector.used_footnote_numbers:
+ break
+
+ # assign new footnote number
+ old_label = cast(nodes.label, footnote[0])
+ old_label.replace_self(nodes.label('', str(num)))
+ if old_label in footnote['names']:
+ footnote['names'].remove(old_label.astext())
+ footnote['names'].append(str(num))
+
+ # update footnote_references by new footnote number
+ docname = footnote['docname']
+ for ref in collector.footnote_refs:
+ if docname == ref['docname'] and footnote['ids'][0] == ref['refid']:
+ ref.remove(ref[0])
+ ref += nodes.Text(str(num))
+
class FootnoteCollector(nodes.NodeVisitor):
"""Collect footnotes and footnote references on the document"""
- def __init__(self, document: nodes.document) ->None:
+ def __init__(self, document: nodes.document) -> None:
self.auto_footnotes: list[nodes.footnote] = []
self.used_footnote_numbers: set[str] = set()
self.footnote_refs: list[nodes.footnote_reference] = []
super().__init__(document)
+ def unknown_visit(self, node: Node) -> None:
+ pass
+
+ def unknown_departure(self, node: Node) -> None:
+ pass
+
+ def visit_footnote(self, node: nodes.footnote) -> None:
+ if node.get('auto'):
+ self.auto_footnotes.append(node)
+ else:
+ for name in node['names']:
+ self.used_footnote_numbers.add(name)
+
+ def visit_footnote_reference(self, node: nodes.footnote_reference) -> None:
+ self.footnote_refs.append(node)
+
class LaTeXFootnoteTransform(SphinxPostTransform):
"""Convert footnote definitions and references to appropriate form to LaTeX.
@@ -216,14 +357,21 @@ class LaTeXFootnoteTransform(SphinxPostTransform):
<row>
...
"""
+
default_priority = 600
- formats = 'latex',
+ formats = ('latex',)
+ def run(self, **kwargs: Any) -> None:
+ footnotes = list(self.document.findall(nodes.footnote))
+ for node in footnotes:
+ node.parent.remove(node)
+
+ visitor = LaTeXFootnoteVisitor(self.document, footnotes)
+ self.document.walkabout(visitor)
-class LaTeXFootnoteVisitor(nodes.NodeVisitor):
- def __init__(self, document: nodes.document, footnotes: list[nodes.
- footnote]) ->None:
+class LaTeXFootnoteVisitor(nodes.NodeVisitor):
+ def __init__(self, document: nodes.document, footnotes: list[nodes.footnote]) -> None:
self.appeared: dict[tuple[str, str], nodes.footnote] = {}
self.footnotes: list[nodes.footnote] = footnotes
self.pendings: list[nodes.footnote] = []
@@ -231,6 +379,108 @@ class LaTeXFootnoteVisitor(nodes.NodeVisitor):
self.restricted: Element | None = None
super().__init__(document)
+ def unknown_visit(self, node: Node) -> None:
+ pass
+
+ def unknown_departure(self, node: Node) -> None:
+ pass
+
+ def restrict(self, node: Element) -> None:
+ if self.restricted is None:
+ self.restricted = node
+
+ def unrestrict(self, node: Element) -> None:
+ if self.restricted == node:
+ self.restricted = None
+ pos = node.parent.index(node)
+ for i, footnote, in enumerate(self.pendings):
+ fntext = footnotetext('', *footnote.children, ids=footnote['ids'])
+ node.parent.insert(pos + i + 1, fntext)
+ self.pendings = []
+
+ def visit_figure(self, node: nodes.figure) -> None:
+ self.restrict(node)
+
+ def depart_figure(self, node: nodes.figure) -> None:
+ self.unrestrict(node)
+
+ def visit_term(self, node: nodes.term) -> None:
+ self.restrict(node)
+
+ def depart_term(self, node: nodes.term) -> None:
+ self.unrestrict(node)
+
+ def visit_caption(self, node: nodes.caption) -> None:
+ self.restrict(node)
+
+ def depart_caption(self, node: nodes.caption) -> None:
+ self.unrestrict(node)
+
+ def visit_title(self, node: nodes.title) -> None:
+ if isinstance(node.parent, nodes.section | nodes.table):
+ self.restrict(node)
+
+ def depart_title(self, node: nodes.title) -> None:
+ if isinstance(node.parent, nodes.section):
+ self.unrestrict(node)
+ elif isinstance(node.parent, nodes.table):
+ self.table_footnotes += self.pendings
+ self.pendings = []
+ self.unrestrict(node)
+
+ def visit_thead(self, node: nodes.thead) -> None:
+ self.restrict(node)
+
+ def depart_thead(self, node: nodes.thead) -> None:
+ self.table_footnotes += self.pendings
+ self.pendings = []
+ self.unrestrict(node)
+
+ def depart_table(self, node: nodes.table) -> None:
+ tbody = next(node.findall(nodes.tbody))
+ for footnote in reversed(self.table_footnotes):
+ fntext = footnotetext('', *footnote.children, ids=footnote['ids'])
+ tbody.insert(0, fntext)
+
+ self.table_footnotes = []
+
+ def visit_footnote(self, node: nodes.footnote) -> None:
+ self.restrict(node)
+
+ def depart_footnote(self, node: nodes.footnote) -> None:
+ self.unrestrict(node)
+
+ def visit_footnote_reference(self, node: nodes.footnote_reference) -> None:
+ number = node.astext().strip()
+ docname = node['docname']
+ if (docname, number) in self.appeared:
+ footnote = self.appeared[(docname, number)]
+ footnote["referred"] = True
+
+ mark = footnotemark('', number, refid=node['refid'])
+ node.replace_self(mark)
+ else:
+ footnote = self.get_footnote_by_reference(node)
+ if self.restricted:
+ mark = footnotemark('', number, refid=node['refid'])
+ node.replace_self(mark)
+ self.pendings.append(footnote)
+ else:
+ self.footnotes.remove(footnote)
+ node.replace_self(footnote)
+ footnote.walkabout(self)
+
+ self.appeared[(docname, number)] = footnote
+ raise nodes.SkipNode
+
+ def get_footnote_by_reference(self, node: nodes.footnote_reference) -> nodes.footnote:
+ docname = node['docname']
+ for footnote in self.footnotes:
+ if docname == footnote['docname'] and footnote['ids'][0] == node['refid']:
+ return footnote
+
+ raise ValueError(__('No footnote was found for given reference node %r') % node)
+
class BibliographyTransform(SphinxPostTransform):
"""Gather bibliography entries to tail of document.
@@ -262,8 +512,18 @@ class BibliographyTransform(SphinxPostTransform):
<citation>
...
"""
+
default_priority = 750
- formats = 'latex',
+ formats = ('latex',)
+
+ def run(self, **kwargs: Any) -> None:
+ citations = thebibliography()
+ for node in list(self.document.findall(nodes.citation)):
+ node.parent.remove(node)
+ citations += node
+
+ if len(citations) > 0:
+ self.document += citations # type: ignore[attr-defined]
class CitationReferenceTransform(SphinxPostTransform):
@@ -272,8 +532,19 @@ class CitationReferenceTransform(SphinxPostTransform):
To handle citation reference easily on LaTeX writer, this converts
pending_xref nodes to citation_reference.
"""
- default_priority = 5
- formats = 'latex',
+
+ default_priority = 5 # before ReferencesResolver
+ formats = ('latex',)
+
+ def run(self, **kwargs: Any) -> None:
+ domain = cast(CitationDomain, self.env.get_domain('citation'))
+ matcher = NodeMatcher(addnodes.pending_xref, refdomain='citation', reftype='ref')
+ for node in matcher.findall(self.document):
+ docname, labelid, _ = domain.citations.get(node['reftarget'], ('', '', 0))
+ if docname:
+ citation_ref = nodes.citation_reference('', '', *node.children,
+ docname=docname, refname=labelid)
+ node.replace_self(citation_ref)
class MathReferenceTransform(SphinxPostTransform):
@@ -282,27 +553,51 @@ class MathReferenceTransform(SphinxPostTransform):
To handle math reference easily on LaTeX writer, this converts pending_xref
nodes to math_reference.
"""
- default_priority = 5
- formats = 'latex',
+
+ default_priority = 5 # before ReferencesResolver
+ formats = ('latex',)
+
+ def run(self, **kwargs: Any) -> None:
+ equations = self.env.get_domain('math').data['objects']
+ for node in self.document.findall(addnodes.pending_xref):
+ if node['refdomain'] == 'math' and node['reftype'] in ('eq', 'numref'):
+ docname, _ = equations.get(node['reftarget'], (None, None))
+ if docname:
+ refnode = math_reference('', docname=docname, target=node['reftarget'])
+ node.replace_self(refnode)
class LiteralBlockTransform(SphinxPostTransform):
"""Replace container nodes for literal_block by captioned_literal_block."""
+
default_priority = 400
- formats = 'latex',
+ formats = ('latex',)
+
+ def run(self, **kwargs: Any) -> None:
+ matcher = NodeMatcher(nodes.container, literal_block=True)
+ for node in matcher.findall(self.document):
+ newnode = captioned_literal_block('', *node.children, **node.attributes)
+ node.replace_self(newnode)
class DocumentTargetTransform(SphinxPostTransform):
"""Add :doc label to the first section of each document."""
+
default_priority = 400
- formats = 'latex',
+ formats = ('latex',)
+
+ def run(self, **kwargs: Any) -> None:
+ for node in self.document.findall(addnodes.start_of_file):
+ section = node.next_node(nodes.section)
+ if section:
+ section['ids'].append(':doc') # special label for :doc:
class IndexInSectionTitleTransform(SphinxPostTransform):
- """Move index nodes in section title to outside of the title.
+ r"""Move index nodes in section title to outside of the title.
LaTeX index macro is not compatible with some handling of section titles
- such as uppercasing done on LaTeX side (cf. fncychap handling of ``\\chapter``).
+ such as uppercasing done on LaTeX side (cf. fncychap handling of ``\chapter``).
Moving the index node to after the title node fixes that.
Before::
@@ -324,5 +619,33 @@ class IndexInSectionTitleTransform(SphinxPostTransform):
blah blah blah
...
"""
+
default_priority = 400
- formats = 'latex',
+ formats = ('latex',)
+
+ def run(self, **kwargs: Any) -> None:
+ for node in list(self.document.findall(nodes.title)):
+ if isinstance(node.parent, nodes.section):
+ for i, index in enumerate(node.findall(addnodes.index)):
+ # move the index node next to the section title
+ node.remove(index)
+ node.parent.insert(i + 1, index)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_transform(FootnoteDocnameUpdater)
+ app.add_post_transform(SubstitutionDefinitionsRemover)
+ app.add_post_transform(BibliographyTransform)
+ app.add_post_transform(CitationReferenceTransform)
+ app.add_post_transform(DocumentTargetTransform)
+ app.add_post_transform(IndexInSectionTitleTransform)
+ app.add_post_transform(LaTeXFootnoteTransform)
+ app.add_post_transform(LiteralBlockTransform)
+ app.add_post_transform(MathReferenceTransform)
+ app.add_post_transform(ShowUrlsTransform)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/latex/util.py b/sphinx/builders/latex/util.py
index b909a8426..aeef26014 100644
--- a/sphinx/builders/latex/util.py
+++ b/sphinx/builders/latex/util.py
@@ -1,18 +1,48 @@
"""Utilities for LaTeX builder."""
+
from __future__ import annotations
+
from docutils.writers.latex2e import Babel
class ExtBabel(Babel):
- cyrillic_languages = ('bulgarian', 'kazakh', 'mongolian', 'russian',
- 'ukrainian')
+ cyrillic_languages = ('bulgarian', 'kazakh', 'mongolian', 'russian', 'ukrainian')
- def __init__(self, language_code: str, use_polyglossia: bool=False) ->None:
+ def __init__(self, language_code: str, use_polyglossia: bool = False) -> None:
self.language_code = language_code
self.use_polyglossia = use_polyglossia
self.supported = True
super().__init__(language_code)
- def get_mainlanguage_options(self) ->(str | None):
- """Return options for polyglossia's ``\\setmainlanguage``."""
- pass
+ def uses_cyrillic(self) -> bool:
+ return self.language in self.cyrillic_languages
+
+ def is_supported_language(self) -> bool:
+ return self.supported
+
+ def language_name(self, language_code: str) -> str:
+ language = super().language_name(language_code)
+ if language == 'ngerman' and self.use_polyglossia:
+ # polyglossia calls new orthography (Neue Rechtschreibung) as
+ # german (with new spelling option).
+ return 'german'
+ elif language:
+ return language
+ elif language_code.startswith('zh'):
+ return 'english' # fallback to english (behaves like supported)
+ else:
+ self.supported = False
+ return 'english' # fallback to english
+
+ def get_mainlanguage_options(self) -> str | None:
+ r"""Return options for polyglossia's ``\setmainlanguage``."""
+ if self.use_polyglossia is False:
+ return None
+ elif self.language == 'german':
+ language = super().language_name(self.language_code)
+ if language == 'ngerman':
+ return 'spelling=new'
+ else:
+ return 'spelling=old'
+ else:
+ return None
diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py
index 14686b733..e9b07164e 100644
--- a/sphinx/builders/linkcheck.py
+++ b/sphinx/builders/linkcheck.py
@@ -1,5 +1,7 @@
"""The CheckExternalLinksBuilder class."""
+
from __future__ import annotations
+
import contextlib
import json
import re
@@ -11,9 +13,11 @@ from queue import PriorityQueue, Queue
from threading import Thread
from typing import TYPE_CHECKING, NamedTuple, cast
from urllib.parse import quote, unquote, urlparse, urlsplit, urlunparse
+
from docutils import nodes
from requests.exceptions import ConnectionError, HTTPError, SSLError, TooManyRedirects
from requests.exceptions import Timeout as RequestTimeout
+
from sphinx.builders.dummy import DummyBuilder
from sphinx.locale import __
from sphinx.transforms.post_transforms import SphinxPostTransform
@@ -21,18 +25,25 @@ from sphinx.util import encode_uri, logging, requests
from sphinx.util.console import darkgray, darkgreen, purple, red, turquoise
from sphinx.util.http_date import rfc1123_to_epoch
from sphinx.util.nodes import get_node_line
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
from typing import Any
+
from requests import Response
+
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util._pathlib import _StrPath
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
-uri_re = re.compile('([a-z]+:)?//')
-DEFAULT_REQUEST_HEADERS = {'Accept':
- 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8'}
+
+uri_re = re.compile('([a-z]+:)?//') # matches to foo:// and // (a protocol relative URL)
+
+DEFAULT_REQUEST_HEADERS = {
+ 'Accept': 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8',
+}
CHECK_IMMEDIATELY = 0
QUEUE_POLL_SECS = 1
DEFAULT_DELAY = 60.0
@@ -42,16 +53,118 @@ class CheckExternalLinksBuilder(DummyBuilder):
"""
Checks for broken external links.
"""
+
name = 'linkcheck'
- epilog = __(
- 'Look for any errors in the above output or in %(outdir)s/output.txt')
+ epilog = __('Look for any errors in the above output or in '
+ '%(outdir)s/output.txt')
+
+ def init(self) -> None:
+ self.broken_hyperlinks = 0
+ self.timed_out_hyperlinks = 0
+ self.hyperlinks: dict[str, Hyperlink] = {}
+ # set a timeout for non-responding servers
+ socket.setdefaulttimeout(5.0)
+
+ def finish(self) -> None:
+ checker = HyperlinkAvailabilityChecker(self.config)
+ logger.info('')
+
+ output_text = path.join(self.outdir, 'output.txt')
+ output_json = path.join(self.outdir, 'output.json')
+ with open(output_text, 'w', encoding='utf-8') as self.txt_outfile, \
+ open(output_json, 'w', encoding='utf-8') as self.json_outfile:
+ for result in checker.check(self.hyperlinks):
+ self.process_result(result)
+
+ if self.broken_hyperlinks or self.timed_out_hyperlinks:
+ self.app.statuscode = 1
+
+ def process_result(self, result: CheckResult) -> None:
+ filename = self.env.doc2path(result.docname, False)
+
+ linkstat: dict[str, str | int] = {
+ 'filename': str(filename), 'lineno': result.lineno,
+ 'status': result.status, 'code': result.code,
+ 'uri': result.uri, 'info': result.message,
+ }
+ self.write_linkstat(linkstat)
+
+ if result.status == 'unchecked':
+ return
+ if result.status == 'working' and result.message == 'old':
+ return
+ if result.lineno:
+ logger.info('(%16s: line %4d) ', result.docname, result.lineno, nonl=True)
+ if result.status == 'ignored':
+ if result.message:
+ logger.info(darkgray('-ignored- ') + result.uri + ': ' + result.message)
+ else:
+ logger.info(darkgray('-ignored- ') + result.uri)
+ elif result.status == 'local':
+ logger.info(darkgray('-local- ') + result.uri)
+ self.write_entry('local', result.docname, filename, result.lineno, result.uri)
+ elif result.status == 'working':
+ logger.info(darkgreen('ok ') + result.uri + result.message)
+ elif result.status == 'timeout':
+ if self.app.quiet:
+ logger.warning('timeout ' + result.uri + result.message,
+ location=(result.docname, result.lineno))
+ else:
+ logger.info(red('timeout ') + result.uri + red(' - ' + result.message))
+ self.write_entry('timeout', result.docname, filename, result.lineno,
+ result.uri + ': ' + result.message)
+ self.timed_out_hyperlinks += 1
+ elif result.status == 'broken':
+ if self.app.quiet:
+ logger.warning(__('broken link: %s (%s)'), result.uri, result.message,
+ location=(result.docname, result.lineno))
+ else:
+ logger.info(red('broken ') + result.uri + red(' - ' + result.message))
+ self.write_entry('broken', result.docname, filename, result.lineno,
+ result.uri + ': ' + result.message)
+ self.broken_hyperlinks += 1
+ elif result.status == 'redirected':
+ try:
+ text, color = {
+ 301: ('permanently', purple),
+ 302: ('with Found', purple),
+ 303: ('with See Other', purple),
+ 307: ('temporarily', turquoise),
+ 308: ('permanently', purple),
+ }[result.code]
+ except KeyError:
+ text, color = ('with unknown code', purple)
+ linkstat['text'] = text
+ if self.config.linkcheck_allowed_redirects:
+ logger.warning('redirect ' + result.uri + ' - ' + text + ' to ' +
+ result.message, location=(result.docname, result.lineno))
+ else:
+ logger.info(color('redirect ') + result.uri +
+ color(' - ' + text + ' to ' + result.message))
+ self.write_entry('redirected ' + text, result.docname, filename,
+ result.lineno, result.uri + ' to ' + result.message)
+ else:
+ raise ValueError('Unknown status %s.' % result.status)
+
+ def write_linkstat(self, data: dict[str, str | int]) -> None:
+ self.json_outfile.write(json.dumps(data))
+ self.json_outfile.write('\n')
+
+ def write_entry(self, what: str, docname: str, filename: _StrPath, line: int,
+ uri: str) -> None:
+ self.txt_outfile.write(f'{filename}:{line}: [{what}] {uri}\n')
class HyperlinkCollector(SphinxPostTransform):
- builders = 'linkcheck',
+ builders = ('linkcheck',)
default_priority = 800
- def find_uri(self, node: nodes.Element) ->(str | None):
+ def run(self, **kwargs: Any) -> None:
+ for node in self.document.findall():
+ if uri := self.find_uri(node):
+ self._add_uri(uri, node)
+
+ def find_uri(self, node: nodes.Element) -> str | None:
"""Find a URI for a given node.
This call can be used to retrieve a URI from a provided node. If no
@@ -64,9 +177,26 @@ class HyperlinkCollector(SphinxPostTransform):
:param node: A node class
:returns: URI of the node
"""
- pass
-
- def _add_uri(self, uri: str, node: nodes.Element) ->None:
+ # reference nodes
+ if isinstance(node, nodes.reference):
+ if 'refuri' in node:
+ return node['refuri']
+
+ # image nodes
+ if isinstance(node, nodes.image):
+ uri = node['candidates'].get('?')
+ if uri and '://' in uri:
+ return uri
+
+ # raw nodes
+ if isinstance(node, nodes.raw):
+ uri = node.get('source')
+ if uri and '://' in uri:
+ return uri
+
+ return None
+
+ def _add_uri(self, uri: str, node: nodes.Element) -> None:
"""Registers a node's URI into a builder's collection of hyperlinks.
Provides the ability to register a URI value determined from a node
@@ -77,7 +207,20 @@ class HyperlinkCollector(SphinxPostTransform):
:param uri: URI to add
:param node: A node class where the URI was found
"""
- pass
+ builder = cast(CheckExternalLinksBuilder, self.app.builder)
+ hyperlinks = builder.hyperlinks
+ docname = self.env.docname
+
+ if newuri := self.app.emit_firstresult('linkcheck-process-uri', uri):
+ uri = newuri
+
+ try:
+ lineno = get_node_line(node)
+ except ValueError:
+ lineno = -1
+
+ if uri not in hyperlinks:
+ hyperlinks[uri] = Hyperlink(uri, docname, self.env.doc2path(docname), lineno)
class Hyperlink(NamedTuple):
@@ -88,16 +231,51 @@ class Hyperlink(NamedTuple):
class HyperlinkAvailabilityChecker:
-
- def __init__(self, config: Config) ->None:
+ def __init__(self, config: Config) -> None:
self.config = config
self.rate_limits: dict[str, RateLimit] = {}
self.rqueue: Queue[CheckResult] = Queue()
self.workers: list[Thread] = []
self.wqueue: PriorityQueue[CheckRequest] = PriorityQueue()
self.num_workers: int = config.linkcheck_workers
- self.to_ignore: list[re.Pattern[str]] = list(map(re.compile, self.
- config.linkcheck_ignore))
+
+ self.to_ignore: list[re.Pattern[str]] = list(map(re.compile,
+ self.config.linkcheck_ignore))
+
+ def check(self, hyperlinks: dict[str, Hyperlink]) -> Iterator[CheckResult]:
+ self.invoke_threads()
+
+ total_links = 0
+ for hyperlink in hyperlinks.values():
+ if self.is_ignored_uri(hyperlink.uri):
+ yield CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno,
+ 'ignored', '', 0)
+ else:
+ self.wqueue.put(CheckRequest(CHECK_IMMEDIATELY, hyperlink), False)
+ total_links += 1
+
+ done = 0
+ while done < total_links:
+ yield self.rqueue.get()
+ done += 1
+
+ self.shutdown_threads()
+
+ def invoke_threads(self) -> None:
+ for _i in range(self.num_workers):
+ thread = HyperlinkAvailabilityCheckWorker(self.config,
+ self.rqueue, self.wqueue,
+ self.rate_limits)
+ thread.start()
+ self.workers.append(thread)
+
+ def shutdown_threads(self) -> None:
+ self.wqueue.join()
+ for _worker in self.workers:
+ self.wqueue.put(CheckRequest(CHECK_IMMEDIATELY, None), False)
+
+ def is_ignored_uri(self, uri: str) -> bool:
+ return any(pat.match(uri) for pat in self.to_ignore)
class CheckRequest(NamedTuple):
@@ -117,22 +295,25 @@ class CheckResult(NamedTuple):
class HyperlinkAvailabilityCheckWorker(Thread):
"""A worker class for checking the availability of hyperlinks."""
- def __init__(self, config: Config, rqueue: Queue[CheckResult], wqueue:
- Queue[CheckRequest], rate_limits: dict[str, RateLimit]) ->None:
+ def __init__(self, config: Config,
+ rqueue: Queue[CheckResult],
+ wqueue: Queue[CheckRequest],
+ rate_limits: dict[str, RateLimit]) -> None:
self.rate_limits = rate_limits
self.rqueue = rqueue
self.wqueue = wqueue
- self.anchors_ignore: list[re.Pattern[str]] = list(map(re.compile,
- config.linkcheck_anchors_ignore))
- self.anchors_ignore_for_url: list[re.Pattern[str]] = list(map(re.
- compile, config.linkcheck_anchors_ignore_for_url))
- self.documents_exclude: list[re.Pattern[str]] = list(map(re.compile,
- config.linkcheck_exclude_documents))
- self.auth = [(re.compile(pattern), auth_info) for pattern,
- auth_info in config.linkcheck_auth]
+
+ self.anchors_ignore: list[re.Pattern[str]] = list(
+ map(re.compile, config.linkcheck_anchors_ignore))
+ self.anchors_ignore_for_url: list[re.Pattern[str]] = list(
+ map(re.compile, config.linkcheck_anchors_ignore_for_url))
+ self.documents_exclude: list[re.Pattern[str]] = list(
+ map(re.compile, config.linkcheck_exclude_documents))
+ self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
+ in config.linkcheck_auth]
+
self.timeout: int | float | None = config.linkcheck_timeout
- self.request_headers: dict[str, dict[str, str]
- ] = config.linkcheck_request_headers
+ self.request_headers: dict[str, dict[str, str]] = config.linkcheck_request_headers
self.check_anchors: bool = config.linkcheck_anchors
self.allowed_redirects: dict[re.Pattern[str], re.Pattern[str]]
self.allowed_redirects = config.linkcheck_allowed_redirects
@@ -143,41 +324,371 @@ class HyperlinkAvailabilityCheckWorker(Thread):
self._timeout_status = 'broken'
else:
self._timeout_status = 'timeout'
+
self.user_agent = config.user_agent
self.tls_verify = config.tls_verify
self.tls_cacerts = config.tls_cacerts
+
self._session = requests._Session()
+
super().__init__(daemon=True)
+ def run(self) -> None:
+ while True:
+ next_check, hyperlink = self.wqueue.get()
+ if hyperlink is None:
+ # An empty hyperlink is a signal to shutdown the worker; cleanup resources here
+ self._session.close()
+ break
+
+ uri, docname, _docpath, lineno = hyperlink
+ if uri is None:
+ break
+
+ netloc = urlsplit(uri).netloc
+ with contextlib.suppress(KeyError):
+ # Refresh rate limit.
+ # When there are many links in the queue, workers are all stuck waiting
+ # for responses, but the builder keeps queuing. Links in the queue may
+ # have been queued before rate limits were discovered.
+ next_check = self.rate_limits[netloc].next_check
+ if next_check > time.time():
+ # Sleep before putting message back in the queue to avoid
+ # waking up other threads.
+ time.sleep(QUEUE_POLL_SECS)
+ self.wqueue.put(CheckRequest(next_check, hyperlink), False)
+ self.wqueue.task_done()
+ continue
+ status, info, code = self._check(docname, uri, hyperlink)
+ if status == 'rate-limited':
+ logger.info(darkgray('-rate limited- ') + uri + darkgray(' | sleeping...'))
+ else:
+ self.rqueue.put(CheckResult(uri, docname, lineno, status, info, code))
+ self.wqueue.task_done()
+
+ def _check(self, docname: str, uri: str, hyperlink: Hyperlink) -> tuple[str, str, int]:
+ # check for various conditions without bothering the network
+
+ for doc_matcher in self.documents_exclude:
+ if doc_matcher.match(docname):
+ info = (
+ f'{docname} matched {doc_matcher.pattern} from '
+ 'linkcheck_exclude_documents'
+ )
+ return 'ignored', info, 0
+
+ if len(uri) == 0 or uri.startswith(('#', 'mailto:', 'tel:')):
+ return 'unchecked', '', 0
+ if not uri.startswith(('http:', 'https:')):
+ if uri_re.match(uri):
+ # Non-supported URI schemes (ex. ftp)
+ return 'unchecked', '', 0
+
+ src_dir = path.dirname(hyperlink.docpath)
+ if path.exists(path.join(src_dir, uri)):
+ return 'working', '', 0
+ return 'broken', '', 0
+
+ # need to actually check the URI
+ status, info, code = '', '', 0
+ for _ in range(self.retries):
+ status, info, code = self._check_uri(uri, hyperlink)
+ if status != 'broken':
+ break
+
+ return status, info, code
+
+ def _retrieval_methods(
+ self,
+ check_anchors: bool,
+ anchor: str,
+ ) -> Iterator[tuple[Callable[..., Response], dict[str, bool]]]:
+ if not check_anchors or not anchor:
+ yield self._session.head, {'allow_redirects': True}
+ yield self._session.get, {'stream': True}
+
+ def _check_uri(self, uri: str, hyperlink: Hyperlink) -> tuple[str, str, int]:
+ req_url, delimiter, anchor = uri.partition('#')
+ if delimiter and anchor:
+ for rex in self.anchors_ignore:
+ if rex.match(anchor):
+ anchor = ''
+ break
+ else:
+ for rex in self.anchors_ignore_for_url:
+ if rex.match(req_url):
+ anchor = ''
+ break
+ anchor = unquote(anchor)
+
+ # handle non-ASCII URIs
+ try:
+ req_url.encode('ascii')
+ except UnicodeError:
+ req_url = encode_uri(req_url)
+
+ # Get auth info, if any
+ for pattern, auth_info in self.auth: # NoQA: B007 (false positive)
+ if pattern.match(uri):
+ break
+ else:
+ auth_info = None
+
+ # update request headers for the URL
+ headers = _get_request_headers(uri, self.request_headers)
+
+ # Linkcheck HTTP request logic:
+ #
+ # - Attempt HTTP HEAD before HTTP GET unless page content is required.
+ # - Follow server-issued HTTP redirects.
+ # - Respect server-issued HTTP 429 back-offs.
+ error_message = ''
+ status_code = -1
+ response_url = retry_after = ''
+ for retrieval_method, kwargs in self._retrieval_methods(self.check_anchors, anchor):
+ try:
+ with retrieval_method(
+ url=req_url, auth=auth_info,
+ headers=headers,
+ timeout=self.timeout,
+ **kwargs,
+ _user_agent=self.user_agent,
+ _tls_info=(self.tls_verify, self.tls_cacerts),
+ ) as response:
+ if anchor and self.check_anchors and response.ok:
+ try:
+ found = contains_anchor(response, anchor)
+ except UnicodeDecodeError:
+ return 'ignored', 'unable to decode response content', 0
+ if not found:
+ return 'broken', __("Anchor '%s' not found") % quote(anchor), 0
+
+ # Copy data we need from the (closed) response
+ status_code = response.status_code
+ redirect_status_code = response.history[-1].status_code if response.history else None # NoQA: E501
+ retry_after = response.headers.get('Retry-After', '')
+ response_url = f'{response.url}'
+ response.raise_for_status()
+ del response
+ break
+
+ except RequestTimeout as err:
+ return self._timeout_status, str(err), 0
+
+ except SSLError as err:
+ # SSL failure; report that the link is broken.
+ return 'broken', str(err), 0
+
+ except (ConnectionError, TooManyRedirects) as err:
+ # Servers drop the connection on HEAD requests, causing
+ # ConnectionError.
+ error_message = str(err)
+ continue
+
+ except HTTPError as err:
+ error_message = str(err)
+
+ # Unauthorized: the client did not provide required credentials
+ if status_code == 401:
+ status = 'working' if self._allow_unauthorized else 'broken'
+ return status, 'unauthorized', 0
+
+ # Rate limiting; back-off if allowed, or report failure otherwise
+ if status_code == 429:
+ if next_check := self.limit_rate(response_url, retry_after):
+ self.wqueue.put(CheckRequest(next_check, hyperlink), False)
+ return 'rate-limited', '', 0
+ return 'broken', error_message, 0
+
+ # Don't claim success/failure during server-side outages
+ if status_code == 503:
+ return 'ignored', 'service unavailable', 0
+
+ # For most HTTP failures, continue attempting alternate retrieval methods
+ continue
+
+ except Exception as err:
+ # Unhandled exception (intermittent or permanent); report that
+ # the link is broken.
+ return 'broken', str(err), 0
-def contains_anchor(response: Response, anchor: str) ->bool:
+ else:
+ # All available retrieval methods have been exhausted; report
+ # that the link is broken.
+ return 'broken', error_message, 0
+
+ # Success; clear rate limits for the origin
+ netloc = urlsplit(req_url).netloc
+ self.rate_limits.pop(netloc, None)
+
+ if ((response_url.rstrip('/') == req_url.rstrip('/'))
+ or _allowed_redirect(req_url, response_url,
+ self.allowed_redirects)):
+ return 'working', '', 0
+ elif redirect_status_code is not None:
+ return 'redirected', response_url, redirect_status_code
+ else:
+ return 'redirected', response_url, 0
+
+ def limit_rate(self, response_url: str, retry_after: str | None) -> float | None:
+ delay = DEFAULT_DELAY
+ next_check = None
+ if retry_after:
+ try:
+ # Integer: time to wait before next attempt.
+ delay = float(retry_after)
+ except ValueError:
+ try:
+ # An HTTP-date: time of next attempt.
+ next_check = rfc1123_to_epoch(retry_after)
+ except (ValueError, TypeError):
+ # TypeError: Invalid date format.
+ # ValueError: Invalid date, e.g. Oct 52th.
+ pass
+ else:
+ delay = next_check - time.time()
+ else:
+ next_check = time.time() + delay
+ netloc = urlsplit(response_url).netloc
+ if next_check is None:
+ max_delay = self.rate_limit_timeout
+ try:
+ rate_limit = self.rate_limits[netloc]
+ except KeyError:
+ delay = DEFAULT_DELAY
+ else:
+ last_wait_time = rate_limit.delay
+ delay = 2.0 * last_wait_time
+ if delay > max_delay > last_wait_time:
+ delay = max_delay
+ if delay > max_delay:
+ return None
+ next_check = time.time() + delay
+ self.rate_limits[netloc] = RateLimit(delay, next_check)
+ return next_check
+
+
+def _get_request_headers(
+ uri: str,
+ request_headers: dict[str, dict[str, str]],
+) -> dict[str, str]:
+ url = urlsplit(uri)
+ candidates = (f'{url.scheme}://{url.netloc}',
+ f'{url.scheme}://{url.netloc}/',
+ uri,
+ '*')
+
+ for u in candidates:
+ if u in request_headers:
+ return {**DEFAULT_REQUEST_HEADERS, **request_headers[u]}
+ return {}
+
+
+def contains_anchor(response: Response, anchor: str) -> bool:
"""Determine if an anchor is contained within an HTTP response."""
- pass
+ parser = AnchorCheckParser(anchor)
+ # Read file in chunks. If we find a matching anchor, we break
+ # the loop early in hopes not to have to download the whole thing.
+ for chunk in response.iter_content(chunk_size=4096, decode_unicode=True):
+ if isinstance(chunk, bytes): # requests failed to decode
+ chunk = chunk.decode() # manually try to decode it
+
+ parser.feed(chunk)
+ if parser.found:
+ break
+ parser.close()
+ return parser.found
class AnchorCheckParser(HTMLParser):
"""Specialised HTML parser that looks for a specific anchor."""
- def __init__(self, search_anchor: str) ->None:
+ def __init__(self, search_anchor: str) -> None:
super().__init__()
+
self.search_anchor = search_anchor
self.found = False
+ def handle_starttag(self, tag: Any, attrs: Any) -> None:
+ for key, value in attrs:
+ if key in ('id', 'name') and value == self.search_anchor:
+ self.found = True
+ break
+
+
+def _allowed_redirect(url: str, new_url: str,
+ allowed_redirects: dict[re.Pattern[str], re.Pattern[str]]) -> bool:
+ return any(
+ from_url.match(url) and to_url.match(new_url)
+ for from_url, to_url
+ in allowed_redirects.items()
+ )
+
class RateLimit(NamedTuple):
delay: float
next_check: float
-def rewrite_github_anchor(app: Sphinx, uri: str) ->(str | None):
+def rewrite_github_anchor(app: Sphinx, uri: str) -> str | None:
"""Rewrite anchor name of the hyperlink to github.com
The hyperlink anchors in github.com are dynamically generated. This rewrites
them before checking and makes them comparable.
"""
- pass
+ parsed = urlparse(uri)
+ if parsed.hostname == 'github.com' and parsed.fragment:
+ prefixed = parsed.fragment.startswith('user-content-')
+ if not prefixed:
+ fragment = f'user-content-{parsed.fragment}'
+ return urlunparse(parsed._replace(fragment=fragment))
+ return None
-def compile_linkcheck_allowed_redirects(app: Sphinx, config: Config) ->None:
+def compile_linkcheck_allowed_redirects(app: Sphinx, config: Config) -> None:
"""Compile patterns in linkcheck_allowed_redirects to the regexp objects."""
- pass
+ for url, pattern in list(app.config.linkcheck_allowed_redirects.items()):
+ try:
+ app.config.linkcheck_allowed_redirects[re.compile(url)] = re.compile(pattern)
+ except re.error as exc:
+ logger.warning(__('Failed to compile regex in linkcheck_allowed_redirects: %r %s'),
+ exc.pattern, exc.msg)
+ finally:
+ # Remove the original regexp-string
+ app.config.linkcheck_allowed_redirects.pop(url)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(CheckExternalLinksBuilder)
+ app.add_post_transform(HyperlinkCollector)
+
+ app.add_config_value('linkcheck_ignore', [], '')
+ app.add_config_value('linkcheck_exclude_documents', [], '')
+ app.add_config_value('linkcheck_allowed_redirects', {}, '')
+ app.add_config_value('linkcheck_auth', [], '')
+ app.add_config_value('linkcheck_request_headers', {}, '')
+ app.add_config_value('linkcheck_retries', 1, '')
+ app.add_config_value('linkcheck_timeout', 30, '', (int, float))
+ app.add_config_value('linkcheck_workers', 5, '')
+ app.add_config_value('linkcheck_anchors', True, '')
+ # Anchors starting with ! are ignored since they are
+ # commonly used for dynamic pages
+ app.add_config_value('linkcheck_anchors_ignore', ['^!'], '')
+ app.add_config_value('linkcheck_anchors_ignore_for_url', (), '', (tuple, list))
+ app.add_config_value('linkcheck_rate_limit_timeout', 300.0, '', (int, float))
+ app.add_config_value('linkcheck_allow_unauthorized', False, '')
+ app.add_config_value('linkcheck_report_timeouts_as_broken', False, '', bool)
+
+ app.add_event('linkcheck-process-uri')
+
+ app.connect('config-inited', compile_linkcheck_allowed_redirects, priority=800)
+
+ # FIXME: Disable URL rewrite handler for github.com temporarily.
+ # ref: https://github.com/sphinx-doc/sphinx/issues/9435
+ # app.connect('linkcheck-process-uri', rewrite_github_anchor)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/manpage.py b/sphinx/builders/manpage.py
index 20bdcc04a..0070b043f 100644
--- a/sphinx/builders/manpage.py
+++ b/sphinx/builders/manpage.py
@@ -1,10 +1,14 @@
"""Manual pages builder."""
+
from __future__ import annotations
+
import warnings
from os import path
from typing import TYPE_CHECKING, Any
+
from docutils.frontend import OptionParser
from docutils.io import FileOutput
+
from sphinx import addnodes
from sphinx.builders import Builder
from sphinx.locale import __
@@ -14,10 +18,12 @@ from sphinx.util.display import progress_message
from sphinx.util.nodes import inline_all_toctrees
from sphinx.util.osutil import ensuredir, make_filename_from_project
from sphinx.writers.manpage import ManualPageTranslator, ManualPageWriter
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
@@ -25,14 +31,99 @@ class ManualPageBuilder(Builder):
"""
Builds groff output in manual page format.
"""
+
name = 'man'
format = 'man'
epilog = __('The manual pages are in %(outdir)s.')
+
default_translator_class = ManualPageTranslator
supported_image_types: list[str] = []
+ def init(self) -> None:
+ if not self.config.man_pages:
+ logger.warning(__('no "man_pages" config value found; no manual pages '
+ 'will be written'))
+
+ def get_outdated_docs(self) -> str | list[str]:
+ return 'all manpages' # for now
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ return ''
+
+ @progress_message(__('writing'))
+ def write(self, *ignored: Any) -> None:
+ docwriter = ManualPageWriter(self)
+ with warnings.catch_warnings():
+ warnings.filterwarnings('ignore', category=DeprecationWarning)
+ # DeprecationWarning: The frontend.OptionParser class will be replaced
+ # by a subclass of argparse.ArgumentParser in Docutils 0.21 or later.
+ docsettings: Any = OptionParser(
+ defaults=self.env.settings,
+ components=(docwriter,),
+ read_config_files=True).get_default_values()
+
+ for info in self.config.man_pages:
+ docname, name, description, authors, section = info
+ if docname not in self.env.all_docs:
+ logger.warning(__('"man_pages" config value references unknown '
+ 'document %s'), docname)
+ continue
+ if isinstance(authors, str):
+ if authors:
+ authors = [authors]
+ else:
+ authors = []
-def default_man_pages(config: Config) ->list[tuple[str, str, str, list[str],
- int]]:
+ docsettings.title = name
+ docsettings.subtitle = description
+ docsettings.authors = authors
+ docsettings.section = section
+
+ if self.config.man_make_section_directory:
+ dirname = 'man%s' % section
+ ensuredir(path.join(self.outdir, dirname))
+ targetname = f'{dirname}/{name}.{section}'
+ else:
+ targetname = f'{name}.{section}'
+
+ logger.info(darkgreen(targetname) + ' { ')
+ destination = FileOutput(
+ destination_path=path.join(self.outdir, targetname),
+ encoding='utf-8')
+
+ tree = self.env.get_doctree(docname)
+ docnames: set[str] = set()
+ largetree = inline_all_toctrees(self, docnames, docname, tree,
+ darkgreen, [docname])
+ largetree.settings = docsettings
+ logger.info('} ', nonl=True)
+ self.env.resolve_references(largetree, docname, self)
+ # remove pending_xref nodes
+ for pendingnode in largetree.findall(addnodes.pending_xref):
+ pendingnode.replace_self(pendingnode.children)
+
+ docwriter.write(largetree, destination)
+
+ def finish(self) -> None:
+ pass
+
+
+def default_man_pages(config: Config) -> list[tuple[str, str, str, list[str], int]]:
"""Better default man_pages settings."""
- pass
+ filename = make_filename_from_project(config.project)
+ return [(config.root_doc, filename, f'{config.project} {config.release}',
+ [config.author], 1)]
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(ManualPageBuilder)
+
+ app.add_config_value('man_pages', default_man_pages, '')
+ app.add_config_value('man_show_urls', False, '')
+ app.add_config_value('man_make_section_directory', False, '')
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/singlehtml.py b/sphinx/builders/singlehtml.py
index 91dcbb65b..4d58d554c 100644
--- a/sphinx/builders/singlehtml.py
+++ b/sphinx/builders/singlehtml.py
@@ -1,8 +1,12 @@
"""Single HTML builders."""
+
from __future__ import annotations
+
from os import path
from typing import TYPE_CHECKING, Any
+
from docutils import nodes
+
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.environment.adapters.toctree import global_toctree_for_doc
from sphinx.locale import __
@@ -10,10 +14,13 @@ from sphinx.util import logging
from sphinx.util.console import darkgreen
from sphinx.util.display import progress_message
from sphinx.util.nodes import inline_all_toctrees
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
@@ -22,6 +29,176 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
A StandaloneHTMLBuilder subclass that puts the whole document tree on one
HTML page.
"""
+
name = 'singlehtml'
epilog = __('The HTML page is in %(outdir)s.')
+
copysource = False
+
+ def get_outdated_docs(self) -> str | list[str]: # type: ignore[override]
+ return 'all documents'
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ if docname in self.env.all_docs:
+ # all references are on the same page...
+ return '#document-' + docname
+ else:
+ # chances are this is a html_additional_page
+ return docname + self.out_suffix
+
+ def get_relative_uri(self, from_: str, to: str, typ: str | None = None) -> str:
+ # ignore source
+ return self.get_target_uri(to, typ)
+
+ def fix_refuris(self, tree: Node) -> None:
+ # fix refuris with double anchor
+ for refnode in tree.findall(nodes.reference):
+ if 'refuri' not in refnode:
+ continue
+ refuri = refnode['refuri']
+ hashindex = refuri.find('#')
+ if hashindex < 0:
+ continue
+ hashindex = refuri.find('#', hashindex + 1)
+ if hashindex >= 0:
+ # all references are on the same page...
+ refnode['refuri'] = refuri[hashindex:]
+
+ def _get_local_toctree(self, docname: str, collapse: bool = True, **kwargs: Any) -> str:
+ if isinstance(includehidden := kwargs.get('includehidden'), str):
+ if includehidden.lower() == 'false':
+ kwargs['includehidden'] = False
+ elif includehidden.lower() == 'true':
+ kwargs['includehidden'] = True
+ if kwargs.get('maxdepth') == '':
+ kwargs.pop('maxdepth')
+ toctree = global_toctree_for_doc(self.env, docname, self, collapse=collapse, **kwargs)
+ if toctree is not None:
+ self.fix_refuris(toctree)
+ return self.render_partial(toctree)['fragment']
+
+ def assemble_doctree(self) -> nodes.document:
+ master = self.config.root_doc
+ tree = self.env.get_doctree(master)
+ logger.info(darkgreen(master))
+ tree = inline_all_toctrees(self, set(), master, tree, darkgreen, [master])
+ tree['docname'] = master
+ self.env.resolve_references(tree, master, self)
+ self.fix_refuris(tree)
+ return tree
+
+ def assemble_toc_secnumbers(self) -> dict[str, dict[str, tuple[int, ...]]]:
+ # Assemble toc_secnumbers to resolve section numbers on SingleHTML.
+ # Merge all secnumbers to single secnumber.
+ #
+ # Note: current Sphinx has refid confliction in singlehtml mode.
+ # To avoid the problem, it replaces key of secnumbers to
+ # tuple of docname and refid.
+ #
+ # There are related codes in inline_all_toctres() and
+ # HTMLTranslter#add_secnumber().
+ new_secnumbers: dict[str, tuple[int, ...]] = {}
+ for docname, secnums in self.env.toc_secnumbers.items():
+ for id, secnum in secnums.items():
+ alias = f"{docname}/{id}"
+ new_secnumbers[alias] = secnum
+
+ return {self.config.root_doc: new_secnumbers}
+
+ def assemble_toc_fignumbers(self) -> dict[str, dict[str, dict[str, tuple[int, ...]]]]:
+ # Assemble toc_fignumbers to resolve figure numbers on SingleHTML.
+ # Merge all fignumbers to single fignumber.
+ #
+ # Note: current Sphinx has refid confliction in singlehtml mode.
+ # To avoid the problem, it replaces key of secnumbers to
+ # tuple of docname and refid.
+ #
+ # There are related codes in inline_all_toctres() and
+ # HTMLTranslter#add_fignumber().
+ new_fignumbers: dict[str, dict[str, tuple[int, ...]]] = {}
+ # {'foo': {'figure': {'id2': (2,), 'id1': (1,)}}, 'bar': {'figure': {'id1': (3,)}}}
+ for docname, fignumlist in self.env.toc_fignumbers.items():
+ for figtype, fignums in fignumlist.items():
+ alias = f"{docname}/{figtype}"
+ new_fignumbers.setdefault(alias, {})
+ for id, fignum in fignums.items():
+ new_fignumbers[alias][id] = fignum
+
+ return {self.config.root_doc: new_fignumbers}
+
+ def get_doc_context(self, docname: str, body: str, metatags: str) -> dict[str, Any]:
+ # no relation links...
+ toctree = global_toctree_for_doc(self.env, self.config.root_doc, self, collapse=False)
+ # if there is no toctree, toc is None
+ if toctree:
+ self.fix_refuris(toctree)
+ toc = self.render_partial(toctree)['fragment']
+ display_toc = True
+ else:
+ toc = ''
+ display_toc = False
+ return {
+ 'parents': [],
+ 'prev': None,
+ 'next': None,
+ 'docstitle': None,
+ 'title': self.config.html_title,
+ 'meta': None,
+ 'body': body,
+ 'metatags': metatags,
+ 'rellinks': [],
+ 'sourcename': '',
+ 'toc': toc,
+ 'display_toc': display_toc,
+ }
+
+ def write(self, *ignored: Any) -> None:
+ docnames = self.env.all_docs
+
+ with progress_message(__('preparing documents')):
+ self.prepare_writing(docnames) # type: ignore[arg-type]
+
+ with progress_message(__('assembling single document'), nonl=False):
+ doctree = self.assemble_doctree()
+ self.env.toc_secnumbers = self.assemble_toc_secnumbers()
+ self.env.toc_fignumbers = self.assemble_toc_fignumbers()
+
+ with progress_message(__('writing')):
+ self.write_doc_serialized(self.config.root_doc, doctree)
+ self.write_doc(self.config.root_doc, doctree)
+
+ def finish(self) -> None:
+ self.write_additional_files()
+ self.copy_image_files()
+ self.copy_download_files()
+ self.copy_static_files()
+ self.copy_extra_files()
+ self.write_buildinfo()
+ self.dump_inventory()
+
+ @progress_message(__('writing additional files'))
+ def write_additional_files(self) -> None:
+ # no indices or search pages are supported
+
+ # additional pages from conf.py
+ for pagename, template in self.config.html_additional_pages.items():
+ logger.info(' ' + pagename, nonl=True)
+ self.handle_page(pagename, {}, template)
+
+ if self.config.html_use_opensearch:
+ logger.info(' opensearch', nonl=True)
+ fn = path.join(self.outdir, '_static', 'opensearch.xml')
+ self.handle_page('opensearch', {}, 'opensearch.xml', outfilename=fn)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.setup_extension('sphinx.builders.html')
+
+ app.add_builder(SingleFileHTMLBuilder)
+ app.add_config_value('singlehtml_sidebars', lambda self: self.html_sidebars, 'html')
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/texinfo.py b/sphinx/builders/texinfo.py
index e7442b2f8..b62160f8e 100644
--- a/sphinx/builders/texinfo.py
+++ b/sphinx/builders/texinfo.py
@@ -1,12 +1,16 @@
"""Texinfo builder."""
+
from __future__ import annotations
+
import os
import warnings
from os import path
from typing import TYPE_CHECKING, Any
+
from docutils import nodes
from docutils.frontend import OptionParser
from docutils.io import FileOutput
+
from sphinx import addnodes, package_dir
from sphinx.builders import Builder
from sphinx.environment.adapters.asset import ImageAdapter
@@ -19,12 +23,16 @@ from sphinx.util.docutils import new_document
from sphinx.util.nodes import inline_all_toctrees
from sphinx.util.osutil import SEP, copyfile, ensuredir, make_filename_from_project
from sphinx.writers.texinfo import TexinfoTranslator, TexinfoWriter
+
if TYPE_CHECKING:
from collections.abc import Iterable
+
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
template_dir = os.path.join(package_dir, 'templates', 'texinfo')
@@ -33,20 +41,199 @@ class TexinfoBuilder(Builder):
"""
Builds Texinfo output to create Info documentation.
"""
+
name = 'texinfo'
format = 'texinfo'
epilog = __('The Texinfo files are in %(outdir)s.')
if os.name == 'posix':
- epilog += __(
- """
-Run 'make' in that directory to run these through makeinfo
-(use 'make info' here to do that automatically)."""
- )
- supported_image_types = ['image/png', 'image/jpeg', 'image/gif']
+ epilog += __("\nRun 'make' in that directory to run these through "
+ "makeinfo\n"
+ "(use 'make info' here to do that automatically).")
+
+ supported_image_types = ['image/png', 'image/jpeg',
+ 'image/gif']
default_translator_class = TexinfoTranslator
+ def init(self) -> None:
+ self.docnames: Iterable[str] = []
+ self.document_data: list[tuple[str, str, str, str, str, str, str, bool]] = []
+
+ def get_outdated_docs(self) -> str | list[str]:
+ return 'all documents' # for now
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ if docname not in self.docnames:
+ raise NoUri(docname, typ)
+ return '%' + docname
+
+ def get_relative_uri(self, from_: str, to: str, typ: str | None = None) -> str:
+ # ignore source path
+ return self.get_target_uri(to, typ)
-def default_texinfo_documents(config: Config) ->list[tuple[str, str, str,
- str, str, str, str]]:
+ def init_document_data(self) -> None:
+ preliminary_document_data = [list(x) for x in self.config.texinfo_documents]
+ if not preliminary_document_data:
+ logger.warning(__('no "texinfo_documents" config value found; no documents '
+ 'will be written'))
+ return
+ # assign subdirs to titles
+ self.titles: list[tuple[str, str]] = []
+ for entry in preliminary_document_data:
+ docname = entry[0]
+ if docname not in self.env.all_docs:
+ logger.warning(__('"texinfo_documents" config value references unknown '
+ 'document %s'), docname)
+ continue
+ self.document_data.append(entry) # type: ignore[arg-type]
+ if docname.endswith(SEP + 'index'):
+ docname = docname[:-5]
+ self.titles.append((docname, entry[2]))
+
+ def write(self, *ignored: Any) -> None:
+ self.init_document_data()
+ self.copy_assets()
+ for entry in self.document_data:
+ docname, targetname, title, author = entry[:4]
+ targetname += '.texi'
+ direntry = description = category = ''
+ if len(entry) > 6:
+ direntry, description, category = entry[4:7]
+ toctree_only = False
+ if len(entry) > 7:
+ toctree_only = entry[7]
+ destination = FileOutput(
+ destination_path=path.join(self.outdir, targetname),
+ encoding='utf-8')
+ with progress_message(__("processing %s") % targetname, nonl=False):
+ appendices = self.config.texinfo_appendices or []
+ doctree = self.assemble_doctree(docname, toctree_only, appendices=appendices)
+
+ with progress_message(__("writing")):
+ self.post_process_images(doctree)
+ docwriter = TexinfoWriter(self)
+ with warnings.catch_warnings():
+ warnings.filterwarnings('ignore', category=DeprecationWarning)
+ # DeprecationWarning: The frontend.OptionParser class will be replaced
+ # by a subclass of argparse.ArgumentParser in Docutils 0.21 or later.
+ settings: Any = OptionParser(
+ defaults=self.env.settings,
+ components=(docwriter,),
+ read_config_files=True).get_default_values()
+ settings.author = author
+ settings.title = title
+ settings.texinfo_filename = targetname[:-5] + '.info'
+ settings.texinfo_elements = self.config.texinfo_elements
+ settings.texinfo_dir_entry = direntry or ''
+ settings.texinfo_dir_category = category or ''
+ settings.texinfo_dir_description = description or ''
+ settings.docname = docname
+ doctree.settings = settings
+ docwriter.write(doctree, destination)
+ self.copy_image_files(targetname[:-5])
+
+ def assemble_doctree(
+ self, indexfile: str, toctree_only: bool, appendices: list[str],
+ ) -> nodes.document:
+ self.docnames = {indexfile, *appendices}
+ logger.info(darkgreen(indexfile))
+ tree = self.env.get_doctree(indexfile)
+ tree['docname'] = indexfile
+ if toctree_only:
+ # extract toctree nodes from the tree and put them in a
+ # fresh document
+ new_tree = new_document('<texinfo output>')
+ new_sect = nodes.section()
+ new_sect += nodes.title('<Set title in conf.py>',
+ '<Set title in conf.py>')
+ new_tree += new_sect
+ for node in tree.findall(addnodes.toctree):
+ new_sect += node
+ tree = new_tree
+ largetree = inline_all_toctrees(self, self.docnames, indexfile, tree,
+ darkgreen, [indexfile])
+ largetree['docname'] = indexfile
+ for docname in appendices:
+ appendix = self.env.get_doctree(docname)
+ appendix['docname'] = docname
+ largetree.append(appendix)
+ logger.info('')
+ logger.info(__("resolving references..."))
+ self.env.resolve_references(largetree, indexfile, self)
+ # TODO: add support for external :ref:s
+ for pendingnode in largetree.findall(addnodes.pending_xref):
+ docname = pendingnode['refdocname']
+ sectname = pendingnode['refsectname']
+ newnodes: list[Node] = [nodes.emphasis(sectname, sectname)]
+ for subdir, title in self.titles:
+ if docname.startswith(subdir):
+ newnodes.extend((
+ nodes.Text(_(' (in ')),
+ nodes.emphasis(title, title),
+ nodes.Text(')'),
+ ))
+ break
+ else:
+ pass
+ pendingnode.replace_self(newnodes)
+ return largetree
+
+ def copy_assets(self) -> None:
+ self.copy_support_files()
+
+ def copy_image_files(self, targetname: str) -> None:
+ if self.images:
+ stringify_func = ImageAdapter(self.app.env).get_original_image_uri
+ for src in status_iterator(self.images, __('copying images... '), "brown",
+ len(self.images), self.app.verbosity,
+ stringify_func=stringify_func):
+ dest = self.images[src]
+ try:
+ imagedir = self.outdir / f'{targetname}-figures'
+ ensuredir(imagedir)
+ copyfile(
+ self.srcdir / src,
+ imagedir / dest,
+ force=True,
+ )
+ except Exception as err:
+ logger.warning(__('cannot copy image file %r: %s'),
+ path.join(self.srcdir, src), err)
+
+ def copy_support_files(self) -> None:
+ try:
+ with progress_message(__('copying Texinfo support files')):
+ logger.info('Makefile ', nonl=True)
+ copyfile(
+ os.path.join(template_dir, 'Makefile'),
+ self.outdir / 'Makefile',
+ force=True,
+ )
+ except OSError as err:
+ logger.warning(__("error writing file Makefile: %s"), err)
+
+
+def default_texinfo_documents(
+ config: Config,
+) -> list[tuple[str, str, str, str, str, str, str]]:
"""Better default texinfo_documents settings."""
- pass
+ filename = make_filename_from_project(config.project)
+ return [(config.root_doc, filename, config.project, config.author, filename,
+ 'One line description of project', 'Miscellaneous')]
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(TexinfoBuilder)
+
+ app.add_config_value('texinfo_documents', default_texinfo_documents, '')
+ app.add_config_value('texinfo_appendices', [], '')
+ app.add_config_value('texinfo_elements', {}, '')
+ app.add_config_value('texinfo_domain_indices', True, '', types={set, list})
+ app.add_config_value('texinfo_show_urls', 'footnote', '')
+ app.add_config_value('texinfo_no_detailmenu', False, '')
+ app.add_config_value('texinfo_cross_references', True, '')
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/text.py b/sphinx/builders/text.py
index 04593d223..92544034c 100644
--- a/sphinx/builders/text.py
+++ b/sphinx/builders/text.py
@@ -1,18 +1,30 @@
"""Plain-text Sphinx builder."""
+
from __future__ import annotations
+
from os import path
from typing import TYPE_CHECKING
+
from docutils.io import StringOutput
+
from sphinx.builders import Builder
from sphinx.locale import __
from sphinx.util import logging
-from sphinx.util.osutil import _last_modified_time, ensuredir, os_path
+from sphinx.util.osutil import (
+ _last_modified_time,
+ ensuredir,
+ os_path,
+)
from sphinx.writers.text import TextTranslator, TextWriter
+
if TYPE_CHECKING:
from collections.abc import Iterator
+
from docutils import nodes
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
@@ -20,7 +32,68 @@ class TextBuilder(Builder):
name = 'text'
format = 'text'
epilog = __('The text files are in %(outdir)s.')
+
out_suffix = '.txt'
allow_parallel = True
default_translator_class = TextTranslator
+
current_docname: str | None = None
+
+ def init(self) -> None:
+ # section numbers for headings in the currently visited document
+ self.secnumbers: dict[str, tuple[int, ...]] = {}
+
+ def get_outdated_docs(self) -> Iterator[str]:
+ for docname in self.env.found_docs:
+ if docname not in self.env.all_docs:
+ yield docname
+ continue
+ targetname = path.join(self.outdir, docname + self.out_suffix)
+ try:
+ targetmtime = _last_modified_time(targetname)
+ except Exception:
+ targetmtime = 0
+ try:
+ srcmtime = _last_modified_time(self.env.doc2path(docname))
+ if srcmtime > targetmtime:
+ yield docname
+ except OSError:
+ # source doesn't exist anymore
+ pass
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ return ''
+
+ def prepare_writing(self, docnames: set[str]) -> None:
+ self.writer = TextWriter(self)
+
+ def write_doc(self, docname: str, doctree: nodes.document) -> None:
+ self.current_docname = docname
+ self.secnumbers = self.env.toc_secnumbers.get(docname, {})
+ destination = StringOutput(encoding='utf-8')
+ self.writer.write(doctree, destination)
+ outfilename = path.join(self.outdir, os_path(docname) + self.out_suffix)
+ ensuredir(path.dirname(outfilename))
+ try:
+ with open(outfilename, 'w', encoding='utf-8') as f:
+ f.write(self.writer.output)
+ except OSError as err:
+ logger.warning(__("error writing file %s: %s"), outfilename, err)
+
+ def finish(self) -> None:
+ pass
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(TextBuilder)
+
+ app.add_config_value('text_sectionchars', '*=-~"+`', 'env')
+ app.add_config_value('text_newlines', 'unix', 'env')
+ app.add_config_value('text_add_secnumbers', True, 'env')
+ app.add_config_value('text_secnumber_suffix', '. ', 'env')
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/builders/xml.py b/sphinx/builders/xml.py
index 3a70cad78..6be0ff8d6 100644
--- a/sphinx/builders/xml.py
+++ b/sphinx/builders/xml.py
@@ -1,19 +1,30 @@
"""Docutils-native XML and pseudo-XML builders."""
+
from __future__ import annotations
+
from os import path
from typing import TYPE_CHECKING
+
from docutils import nodes
from docutils.io import StringOutput
from docutils.writers.docutils_xml import XMLTranslator
+
from sphinx.builders import Builder
from sphinx.locale import __
from sphinx.util import logging
-from sphinx.util.osutil import _last_modified_time, ensuredir, os_path
+from sphinx.util.osutil import (
+ _last_modified_time,
+ ensuredir,
+ os_path,
+)
from sphinx.writers.xml import PseudoXMLWriter, XMLWriter
+
if TYPE_CHECKING:
from collections.abc import Iterator
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
@@ -21,22 +32,97 @@ class XMLBuilder(Builder):
"""
Builds Docutils-native XML.
"""
+
name = 'xml'
format = 'xml'
epilog = __('The XML files are in %(outdir)s.')
+
out_suffix = '.xml'
allow_parallel = True
+
_writer_class: type[XMLWriter] | type[PseudoXMLWriter] = XMLWriter
writer: XMLWriter | PseudoXMLWriter
default_translator_class = XMLTranslator
+ def init(self) -> None:
+ pass
+
+ def get_outdated_docs(self) -> Iterator[str]:
+ for docname in self.env.found_docs:
+ if docname not in self.env.all_docs:
+ yield docname
+ continue
+ targetname = path.join(self.outdir, docname + self.out_suffix)
+ try:
+ targetmtime = _last_modified_time(targetname)
+ except Exception:
+ targetmtime = 0
+ try:
+ srcmtime = _last_modified_time(self.env.doc2path(docname))
+ if srcmtime > targetmtime:
+ yield docname
+ except OSError:
+ # source doesn't exist anymore
+ pass
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ return docname
+
+ def prepare_writing(self, docnames: set[str]) -> None:
+ self.writer = self._writer_class(self)
+
+ def write_doc(self, docname: str, doctree: nodes.document) -> None:
+ # work around multiple string % tuple issues in docutils;
+ # replace tuples in attribute values with lists
+ doctree = doctree.deepcopy()
+ for domain in self.env.domains.values():
+ xmlns = "xmlns:" + domain.name
+ doctree[xmlns] = "https://www.sphinx-doc.org/"
+ for node in doctree.findall(nodes.Element):
+ for att, value in node.attributes.items():
+ if isinstance(value, tuple):
+ node.attributes[att] = list(value)
+ value = node.attributes[att]
+ if isinstance(value, list):
+ for i, val in enumerate(value):
+ if isinstance(val, tuple):
+ value[i] = list(val)
+ destination = StringOutput(encoding='utf-8')
+ self.writer.write(doctree, destination)
+ outfilename = path.join(self.outdir, os_path(docname) + self.out_suffix)
+ ensuredir(path.dirname(outfilename))
+ try:
+ with open(outfilename, 'w', encoding='utf-8') as f:
+ f.write(self.writer.output)
+ except OSError as err:
+ logger.warning(__("error writing file %s: %s"), outfilename, err)
+
+ def finish(self) -> None:
+ pass
+
class PseudoXMLBuilder(XMLBuilder):
"""
Builds pseudo-XML for display purposes.
"""
+
name = 'pseudoxml'
format = 'pseudoxml'
epilog = __('The pseudo-XML files are in %(outdir)s.')
+
out_suffix = '.pseudoxml'
+
_writer_class = PseudoXMLWriter
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(XMLBuilder)
+ app.add_builder(PseudoXMLBuilder)
+
+ app.add_config_value('xml_pretty', True, 'env')
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/cmd/build.py b/sphinx/cmd/build.py
index a2aa5c5f4..9f6cf2a33 100644
--- a/sphinx/cmd/build.py
+++ b/sphinx/cmd/build.py
@@ -1,17 +1,21 @@
"""Build documentation from a provided source."""
+
from __future__ import annotations
+
import argparse
import bdb
import contextlib
import locale
import multiprocessing
import os
-import pdb
+import pdb # NoQA: T100
import sys
import traceback
from os import path
from typing import TYPE_CHECKING, Any, TextIO
+
from docutils.utils import SystemMessage
+
import sphinx.locale
from sphinx import __display_version__
from sphinx.application import Sphinx
@@ -22,32 +26,373 @@ from sphinx.util.console import color_terminal, nocolor, red, terminal_safe
from sphinx.util.docutils import docutils_namespace, patch_docutils
from sphinx.util.exceptions import format_exception_cut_frames, save_traceback
from sphinx.util.osutil import ensuredir
+
if TYPE_CHECKING:
from collections.abc import Sequence
from typing import Protocol
-
class SupportsWrite(Protocol):
- pass
+ def write(self, text: str, /) -> int | None:
+ ...
+
+def handle_exception(
+ app: Sphinx | None, args: Any, exception: BaseException, stderr: TextIO = sys.stderr,
+) -> None:
+ if isinstance(exception, bdb.BdbQuit):
+ return
-def jobs_argument(value: str) ->int:
+ if args.pdb:
+ print(red(__('Exception occurred while building, starting debugger:')),
+ file=stderr)
+ traceback.print_exc()
+ pdb.post_mortem(sys.exc_info()[2])
+ else:
+ print(file=stderr)
+ if args.verbosity or args.traceback:
+ exc = sys.exc_info()[1]
+ if isinstance(exc, SphinxParallelError):
+ exc_format = '(Error in parallel process)\n' + exc.traceback
+ print(exc_format, file=stderr)
+ else:
+ traceback.print_exc(None, stderr)
+ print(file=stderr)
+ if isinstance(exception, KeyboardInterrupt):
+ print(__('Interrupted!'), file=stderr)
+ elif isinstance(exception, SystemMessage):
+ print(red(__('reST markup error:')), file=stderr)
+ print(terminal_safe(exception.args[0]), file=stderr)
+ elif isinstance(exception, SphinxError):
+ print(red('%s:' % exception.category), file=stderr)
+ print(str(exception), file=stderr)
+ elif isinstance(exception, UnicodeError):
+ print(red(__('Encoding error:')), file=stderr)
+ print(terminal_safe(str(exception)), file=stderr)
+ tbpath = save_traceback(app, exception)
+ print(red(__('The full traceback has been saved in %s, if you want '
+ 'to report the issue to the developers.') % tbpath),
+ file=stderr)
+ elif isinstance(exception, RuntimeError) and 'recursion depth' in str(exception):
+ print(red(__('Recursion error:')), file=stderr)
+ print(terminal_safe(str(exception)), file=stderr)
+ print(file=stderr)
+ print(__('This can happen with very large or deeply nested source '
+ 'files. You can carefully increase the default Python '
+ 'recursion limit of 1000 in conf.py with e.g.:'), file=stderr)
+ print(' import sys; sys.setrecursionlimit(1500)', file=stderr)
+ else:
+ print(red(__('Exception occurred:')), file=stderr)
+ print(format_exception_cut_frames().rstrip(), file=stderr)
+ tbpath = save_traceback(app, exception)
+ print(red(__('The full traceback has been saved in %s, if you '
+ 'want to report the issue to the developers.') % tbpath),
+ file=stderr)
+ print(__('Please also report this if it was a user error, so '
+ 'that a better error message can be provided next time.'),
+ file=stderr)
+ print(__('A bug report can be filed in the tracker at '
+ '<https://github.com/sphinx-doc/sphinx/issues>. Thanks!'),
+ file=stderr)
+
+
+def jobs_argument(value: str) -> int:
"""
Special type to handle 'auto' flags passed to 'sphinx-build' via -j flag. Can
be expanded to handle other special scaling requests, such as setting job count
to cpu_count.
"""
- pass
+ if value == 'auto':
+ return multiprocessing.cpu_count()
+ else:
+ jobs = int(value)
+ if jobs <= 0:
+ raise argparse.ArgumentTypeError(__('job number should be a positive number'))
+ else:
+ return jobs
+
+
+def get_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ usage='%(prog)s [OPTIONS] SOURCEDIR OUTPUTDIR [FILENAMES...]',
+ epilog=__('For more information, visit <https://www.sphinx-doc.org/>.'),
+ description=__("""
+Generate documentation from source files.
+
+sphinx-build generates documentation from the files in SOURCEDIR and places it
+in OUTPUTDIR. It looks for 'conf.py' in SOURCEDIR for the configuration
+settings. The 'sphinx-quickstart' tool may be used to generate template files,
+including 'conf.py'
+
+sphinx-build can create documentation in different formats. A format is
+selected by specifying the builder name on the command line; it defaults to
+HTML. Builders can also perform other tasks related to documentation
+processing.
+
+By default, everything that is outdated is built. Output only for selected
+files can be built by specifying individual filenames.
+"""))
+ parser.add_argument('--version', action='version', dest='show_version',
+ version=f'%(prog)s {__display_version__}')
-def make_main(argv: Sequence[str]) ->int:
+ parser.add_argument('sourcedir', metavar='SOURCE_DIR',
+ help=__('path to documentation source files'))
+ parser.add_argument('outputdir', metavar='OUTPUT_DIR',
+ help=__('path to output directory'))
+ parser.add_argument('filenames', nargs='*',
+ help=__('(optional) a list of specific files to rebuild. '
+ 'Ignored if --write-all is specified'))
+
+ group = parser.add_argument_group(__('general options'))
+ group.add_argument('--builder', '-b', metavar='BUILDER', dest='builder',
+ default='html',
+ help=__("builder to use (default: 'html')"))
+ group.add_argument('--jobs', '-j', metavar='N', default=1, type=jobs_argument,
+ dest='jobs',
+ help=__('run in parallel with N processes, when possible. '
+ "'auto' uses the number of CPU cores"))
+ group.add_argument('--write-all', '-a', action='store_true', dest='force_all',
+ help=__('write all files (default: only write new and '
+ 'changed files)'))
+ group.add_argument('--fresh-env', '-E', action='store_true', dest='freshenv',
+ help=__("don't use a saved environment, always read "
+ 'all files'))
+
+ group = parser.add_argument_group(__('path options'))
+ group.add_argument('--doctree-dir', '-d', metavar='PATH', dest='doctreedir',
+ help=__('directory for doctree and environment files '
+ '(default: OUTPUT_DIR/.doctrees)'))
+ group.add_argument('--conf-dir', '-c', metavar='PATH', dest='confdir',
+ help=__('directory for the configuration file (conf.py) '
+ '(default: SOURCE_DIR)'))
+
+ group = parser.add_argument_group('build configuration options')
+ group.add_argument('--isolated', '-C', action='store_true', dest='noconfig',
+ help=__('use no configuration file, only use settings from -D options'))
+ group.add_argument('--define', '-D', metavar='setting=value', action='append',
+ dest='define', default=[],
+ help=__('override a setting in configuration file'))
+ group.add_argument('--html-define', '-A', metavar='name=value', action='append',
+ dest='htmldefine', default=[],
+ help=__('pass a value into HTML templates'))
+ group.add_argument('--tag', '-t', metavar='TAG', action='append',
+ dest='tags', default=[],
+ help=__('define tag: include "only" blocks with TAG'))
+ group.add_argument('--nitpicky', '-n', action='store_true', dest='nitpicky',
+ help=__('nitpicky mode: warn about all missing references'))
+
+ group = parser.add_argument_group(__('console output options'))
+ group.add_argument('--verbose', '-v', action='count', dest='verbosity',
+ default=0,
+ help=__('increase verbosity (can be repeated)'))
+ group.add_argument('--quiet', '-q', action='store_true', dest='quiet',
+ help=__('no output on stdout, just warnings on stderr'))
+ group.add_argument('--silent', '-Q', action='store_true', dest='really_quiet',
+ help=__('no output at all, not even warnings'))
+ group.add_argument('--color', action='store_const', dest='color',
+ const='yes', default='auto',
+ help=__('do emit colored output (default: auto-detect)'))
+ group.add_argument('--no-color', '-N', action='store_const', dest='color',
+ const='no',
+ help=__('do not emit colored output (default: auto-detect)'))
+
+ group = parser.add_argument_group(__('warning control options'))
+ group.add_argument('--warning-file', '-w', metavar='FILE', dest='warnfile',
+ help=__('write warnings (and errors) to given file'))
+ group.add_argument('--fail-on-warning', '-W', action='store_true', dest='warningiserror',
+ help=__('turn warnings into errors'))
+ group.add_argument('--keep-going', action='store_true', help=argparse.SUPPRESS)
+ group.add_argument('--show-traceback', '-T', action='store_true', dest='traceback',
+ help=__('show full traceback on exception'))
+ group.add_argument('--pdb', '-P', action='store_true', dest='pdb',
+ help=__('run Pdb on exception'))
+ group.add_argument('--exception-on-warning', action='store_true',
+ dest='exception_on_warning',
+ help=__('raise an exception on warnings'))
+
+ if parser.prog == '__main__.py':
+ parser.prog = 'sphinx-build'
+
+ return parser
+
+
+def make_main(argv: Sequence[str]) -> int:
"""Sphinx build "make mode" entry."""
- pass
+ from sphinx.cmd import make_mode
+ return make_mode.run_make_mode(argv[1:])
+
+
+def _parse_arguments(parser: argparse.ArgumentParser,
+ argv: Sequence[str]) -> argparse.Namespace:
+ args = parser.parse_args(argv)
+ return args
+
+
+def _parse_confdir(noconfig: bool, confdir: str, sourcedir: str) -> str | None:
+ if noconfig:
+ return None
+ elif not confdir:
+ return sourcedir
+ return confdir
+
+def _parse_doctreedir(doctreedir: str, outputdir: str) -> str:
+ if doctreedir:
+ return doctreedir
+ return os.path.join(outputdir, '.doctrees')
-def build_main(argv: Sequence[str]) ->int:
+
+def _validate_filenames(
+ parser: argparse.ArgumentParser, force_all: bool, filenames: list[str],
+) -> None:
+ if force_all and filenames:
+ parser.error(__('cannot combine -a option and filenames'))
+
+
+def _validate_colour_support(colour: str) -> None:
+ if colour == 'no' or (colour == 'auto' and not color_terminal()):
+ nocolor()
+
+
+def _parse_logging(
+ parser: argparse.ArgumentParser,
+ quiet: bool,
+ really_quiet: bool,
+ warnfile: str | None,
+) -> tuple[TextIO | None, TextIO | None, TextIO, TextIO | None]:
+ status: TextIO | None = sys.stdout
+ warning: TextIO | None = sys.stderr
+ error = sys.stderr
+
+ if quiet:
+ status = None
+
+ if really_quiet:
+ status = warning = None
+
+ warnfp = None
+ if warning and warnfile:
+ try:
+ warnfile = path.abspath(warnfile)
+ ensuredir(path.dirname(warnfile))
+ # the caller is responsible for closing this file descriptor
+ warnfp = open(warnfile, 'w', encoding="utf-8") # NoQA: SIM115
+ except Exception as exc:
+ parser.error(__('cannot open warning file %r: %s') % (
+ warnfile, exc))
+ warning = TeeStripANSI(warning, warnfp) # type: ignore[assignment]
+ error = warning
+
+ return status, warning, error, warnfp
+
+
+def _parse_confoverrides(
+ parser: argparse.ArgumentParser,
+ define: list[str],
+ htmldefine: list[str],
+ nitpicky: bool,
+) -> dict[str, Any]:
+ confoverrides: dict[str, Any] = {}
+ val: Any
+ for val in define:
+ try:
+ key, val = val.split('=', 1)
+ except ValueError:
+ parser.error(__('-D option argument must be in the form name=value'))
+ confoverrides[key] = val
+
+ for val in htmldefine:
+ try:
+ key, val = val.split('=')
+ except ValueError:
+ parser.error(__('-A option argument must be in the form name=value'))
+ with contextlib.suppress(ValueError):
+ val = int(val)
+
+ confoverrides[f'html_context.{key}'] = val
+
+ if nitpicky:
+ confoverrides['nitpicky'] = True
+
+ return confoverrides
+
+
+def build_main(argv: Sequence[str]) -> int:
"""Sphinx build "main" command-line entry."""
- pass
+ parser = get_parser()
+ args = _parse_arguments(parser, argv)
+ args.confdir = _parse_confdir(args.noconfig, args.confdir, args.sourcedir)
+ args.doctreedir = _parse_doctreedir(args.doctreedir, args.outputdir)
+ _validate_filenames(parser, args.force_all, args.filenames)
+ _validate_colour_support(args.color)
+ args.status, args.warning, args.error, warnfp = _parse_logging(
+ parser, args.quiet, args.really_quiet, args.warnfile)
+ args.confoverrides = _parse_confoverrides(
+ parser, args.define, args.htmldefine, args.nitpicky)
+
+ app = None
+ try:
+ confdir = args.confdir or args.sourcedir
+ with patch_docutils(confdir), docutils_namespace():
+ app = Sphinx(
+ srcdir=args.sourcedir, confdir=args.confdir,
+ outdir=args.outputdir, doctreedir=args.doctreedir,
+ buildername=args.builder, confoverrides=args.confoverrides,
+ status=args.status, warning=args.warning,
+ freshenv=args.freshenv, warningiserror=args.warningiserror,
+ tags=args.tags,
+ verbosity=args.verbosity, parallel=args.jobs, keep_going=False,
+ pdb=args.pdb, exception_on_warning=args.exception_on_warning,
+ )
+ app.build(args.force_all, args.filenames)
+ return app.statuscode
+ except (Exception, KeyboardInterrupt) as exc:
+ handle_exception(app, args, exc, args.error)
+ return 2
+ finally:
+ if warnfp is not None:
+ # close the file descriptor for the warnings file opened by Sphinx
+ warnfp.close()
+
+
+def _bug_report_info() -> int:
+ from platform import platform, python_implementation
+
+ import docutils
+ import jinja2
+ import pygments
+
+ print('Please paste all output below into the bug report template\n\n')
+ print('```text')
+ print(f'Platform: {sys.platform}; ({platform()})')
+ print(f'Python version: {sys.version})')
+ print(f'Python implementation: {python_implementation()}')
+ print(f'Sphinx version: {sphinx.__display_version__}')
+ print(f'Docutils version: {docutils.__version__}')
+ print(f'Jinja2 version: {jinja2.__version__}')
+ print(f'Pygments version: {pygments.__version__}')
+ print('```')
+ return 0
+
+
+def main(argv: Sequence[str] = (), /) -> int:
+ locale.setlocale(locale.LC_ALL, '')
+ sphinx.locale.init_console()
+
+ if not argv:
+ argv = sys.argv[1:]
+
+ # Allow calling as 'python -m sphinx build …'
+ if argv[:1] == ['build']:
+ argv = argv[1:]
+
+ if argv[:1] == ['--bug-report']:
+ return _bug_report_info()
+ if argv[:1] == ['-M']:
+ from sphinx.cmd import make_mode
+ return make_mode.run_make_mode(argv[1:])
+ else:
+ return build_main(argv)
if __name__ == '__main__':
diff --git a/sphinx/cmd/make_mode.py b/sphinx/cmd/make_mode.py
index f817e7f32..d1ba3fccf 100644
--- a/sphinx/cmd/make_mode.py
+++ b/sphinx/cmd/make_mode.py
@@ -6,51 +6,201 @@ of Makefile / make.bat.
This is in its own module so that importing it is fast. It should not
import the main Sphinx modules (like sphinx.applications, sphinx.builders).
"""
+
from __future__ import annotations
+
import os
import subprocess
import sys
from os import path
from typing import TYPE_CHECKING
+
import sphinx
from sphinx.cmd.build import build_main
from sphinx.util.console import blue, bold, color_terminal, nocolor
from sphinx.util.osutil import rmtree
+
if sys.version_info >= (3, 11):
from contextlib import chdir
else:
from sphinx.util.osutil import _chdir as chdir
+
if TYPE_CHECKING:
from collections.abc import Sequence
-BUILDERS = [('', 'html', 'to make standalone HTML files'), ('', 'dirhtml',
- 'to make HTML files named index.html in directories'), ('',
- 'singlehtml', 'to make a single large HTML file'), ('', 'pickle',
- 'to make pickle files'), ('', 'json', 'to make JSON files'), ('',
- 'htmlhelp', 'to make HTML files and an HTML help project'), ('',
- 'qthelp', 'to make HTML files and a qthelp project'), ('', 'devhelp',
- 'to make HTML files and a Devhelp project'), ('', 'epub',
- 'to make an epub'), ('', 'latex',
- 'to make LaTeX files, you can set PAPER=a4 or PAPER=letter'), ('posix',
- 'latexpdf', 'to make LaTeX and PDF files (default pdflatex)'), ('posix',
- 'latexpdfja',
- 'to make LaTeX files and run them through platex/dvipdfmx'), ('',
- 'text', 'to make text files'), ('', 'man', 'to make manual pages'), ('',
- 'texinfo', 'to make Texinfo files'), ('posix', 'info',
- 'to make Texinfo files and run them through makeinfo'), ('', 'gettext',
- 'to make PO message catalogs'), ('', 'changes',
- 'to make an overview of all changed/added/deprecated items'), ('',
- 'xml', 'to make Docutils-native XML files'), ('', 'pseudoxml',
- 'to make pseudoxml-XML files for display purposes'), ('', 'linkcheck',
- 'to check all external links for integrity'), ('', 'doctest',
- 'to run all doctests embedded in the documentation (if enabled)'), ('',
- 'coverage', 'to run coverage check of the documentation (if enabled)'),
- ('', 'clean', 'to remove everything in the build directory')]
+BUILDERS = [
+ ("", "html", "to make standalone HTML files"),
+ ("", "dirhtml", "to make HTML files named index.html in directories"),
+ ("", "singlehtml", "to make a single large HTML file"),
+ ("", "pickle", "to make pickle files"),
+ ("", "json", "to make JSON files"),
+ ("", "htmlhelp", "to make HTML files and an HTML help project"),
+ ("", "qthelp", "to make HTML files and a qthelp project"),
+ ("", "devhelp", "to make HTML files and a Devhelp project"),
+ ("", "epub", "to make an epub"),
+ ("", "latex", "to make LaTeX files, you can set PAPER=a4 or PAPER=letter"),
+ ("posix", "latexpdf", "to make LaTeX and PDF files (default pdflatex)"),
+ ("posix", "latexpdfja", "to make LaTeX files and run them through platex/dvipdfmx"),
+ ("", "text", "to make text files"),
+ ("", "man", "to make manual pages"),
+ ("", "texinfo", "to make Texinfo files"),
+ ("posix", "info", "to make Texinfo files and run them through makeinfo"),
+ ("", "gettext", "to make PO message catalogs"),
+ ("", "changes", "to make an overview of all changed/added/deprecated items"),
+ ("", "xml", "to make Docutils-native XML files"),
+ ("", "pseudoxml", "to make pseudoxml-XML files for display purposes"),
+ ("", "linkcheck", "to check all external links for integrity"),
+ ("", "doctest", "to run all doctests embedded in the documentation "
+ "(if enabled)"),
+ ("", "coverage", "to run coverage check of the documentation (if enabled)"),
+ ("", "clean", "to remove everything in the build directory"),
+]
-class Make:
- def __init__(self, *, source_dir: str, build_dir: str, opts: Sequence[str]
- ) ->None:
+class Make:
+ def __init__(self, *, source_dir: str, build_dir: str, opts: Sequence[str]) -> None:
self.source_dir = source_dir
self.build_dir = build_dir
self.opts = [*opts]
+
+ def build_dir_join(self, *comps: str) -> str:
+ return path.join(self.build_dir, *comps)
+
+ def build_clean(self) -> int:
+ source_dir = path.abspath(self.source_dir)
+ build_dir = path.abspath(self.build_dir)
+ if not path.exists(self.build_dir):
+ return 0
+ elif not path.isdir(self.build_dir):
+ print("Error: %r is not a directory!" % self.build_dir)
+ return 1
+ elif source_dir == build_dir:
+ print("Error: %r is same as source directory!" % self.build_dir)
+ return 1
+ elif path.commonpath([source_dir, build_dir]) == build_dir:
+ print("Error: %r directory contains source directory!" % self.build_dir)
+ return 1
+ print("Removing everything under %r..." % self.build_dir)
+ for item in os.listdir(self.build_dir):
+ rmtree(self.build_dir_join(item))
+ return 0
+
+ def build_help(self) -> None:
+ if not color_terminal():
+ nocolor()
+
+ print(bold("Sphinx v%s" % sphinx.__display_version__))
+ print("Please use `make %s' where %s is one of" % ((blue('target'),) * 2))
+ for osname, bname, description in BUILDERS:
+ if not osname or os.name == osname:
+ print(f' {blue(bname.ljust(10))} {description}')
+
+ def build_latexpdf(self) -> int:
+ if self.run_generic_build('latex') > 0:
+ return 1
+
+ # Use $MAKE to determine the make command
+ make_fallback = 'make.bat' if sys.platform == 'win32' else 'make'
+ makecmd = os.environ.get('MAKE', make_fallback)
+ if not makecmd.lower().startswith('make'):
+ raise RuntimeError('Invalid $MAKE command: %r' % makecmd)
+ try:
+ with chdir(self.build_dir_join('latex')):
+ if '-Q' in self.opts:
+ with open('__LATEXSTDOUT__', 'w') as outfile:
+ returncode = subprocess.call([makecmd,
+ 'all-pdf',
+ 'LATEXOPTS=-halt-on-error',
+ ],
+ stdout=outfile,
+ stderr=subprocess.STDOUT,
+ )
+ if returncode:
+ print('Latex error: check %s' %
+ self.build_dir_join('latex', '__LATEXSTDOUT__')
+ )
+ elif '-q' in self.opts:
+ returncode = subprocess.call(
+ [makecmd,
+ 'all-pdf',
+ 'LATEXOPTS=-halt-on-error',
+ 'LATEXMKOPTS=-silent',
+ ],
+ )
+ if returncode:
+ print('Latex error: check .log file in %s' %
+ self.build_dir_join('latex')
+ )
+ else:
+ returncode = subprocess.call([makecmd, 'all-pdf'])
+ return returncode
+ except OSError:
+ print('Error: Failed to run: %s' % makecmd)
+ return 1
+
+ def build_latexpdfja(self) -> int:
+ if self.run_generic_build('latex') > 0:
+ return 1
+
+ # Use $MAKE to determine the make command
+ make_fallback = 'make.bat' if sys.platform == 'win32' else 'make'
+ makecmd = os.environ.get('MAKE', make_fallback)
+ if not makecmd.lower().startswith('make'):
+ raise RuntimeError('Invalid $MAKE command: %r' % makecmd)
+ try:
+ with chdir(self.build_dir_join('latex')):
+ return subprocess.call([makecmd, 'all-pdf'])
+ except OSError:
+ print('Error: Failed to run: %s' % makecmd)
+ return 1
+
+ def build_info(self) -> int:
+ if self.run_generic_build('texinfo') > 0:
+ return 1
+
+ # Use $MAKE to determine the make command
+ makecmd = os.environ.get('MAKE', 'make')
+ if not makecmd.lower().startswith('make'):
+ raise RuntimeError('Invalid $MAKE command: %r' % makecmd)
+ try:
+ with chdir(self.build_dir_join('texinfo')):
+ return subprocess.call([makecmd, 'info'])
+ except OSError:
+ print('Error: Failed to run: %s' % makecmd)
+ return 1
+
+ def build_gettext(self) -> int:
+ dtdir = self.build_dir_join('gettext', '.doctrees')
+ if self.run_generic_build('gettext', doctreedir=dtdir) > 0:
+ return 1
+ return 0
+
+ def run_generic_build(self, builder: str, doctreedir: str | None = None) -> int:
+ # compatibility with old Makefile
+ paper_size = os.getenv('PAPER', '')
+ if paper_size in {'a4', 'letter'}:
+ self.opts.extend(['-D', f'latex_elements.papersize={paper_size}paper'])
+ if doctreedir is None:
+ doctreedir = self.build_dir_join('doctrees')
+
+ args = [
+ '--builder', builder,
+ '--doctree-dir', doctreedir,
+ self.source_dir,
+ self.build_dir_join(builder),
+ ]
+ return build_main(args + self.opts)
+
+
+def run_make_mode(args: Sequence[str]) -> int:
+ if len(args) < 3:
+ print('Error: at least 3 arguments (builder, source '
+ 'dir, build dir) are required.', file=sys.stderr)
+ return 1
+
+ builder_name = args[0]
+ make = Make(source_dir=args[1], build_dir=args[2], opts=args[3:])
+ run_method = f'build_{builder_name}'
+ if hasattr(make, run_method):
+ return getattr(make, run_method)()
+ return make.run_generic_build(builder_name)
diff --git a/sphinx/cmd/quickstart.py b/sphinx/cmd/quickstart.py
index c9bb73784..4ae4556ca 100644
--- a/sphinx/cmd/quickstart.py
+++ b/sphinx/cmd/quickstart.py
@@ -1,5 +1,7 @@
"""Quickly setup documentation source to work with Sphinx."""
+
from __future__ import annotations
+
import argparse
import locale
import os
@@ -7,70 +9,184 @@ import sys
import time
from os import path
from typing import TYPE_CHECKING, Any
+
+# try to import readline, unix specific enhancement
try:
import readline
- if TYPE_CHECKING and sys.platform == 'win32':
+ if TYPE_CHECKING and sys.platform == "win32": # always false, for type checking
raise ImportError
READLINE_AVAILABLE = True
if readline.__doc__ and 'libedit' in readline.__doc__:
- readline.parse_and_bind('bind ^I rl_complete')
+ readline.parse_and_bind("bind ^I rl_complete")
USE_LIBEDIT = True
else:
- readline.parse_and_bind('tab: complete')
+ readline.parse_and_bind("tab: complete")
USE_LIBEDIT = False
except ImportError:
READLINE_AVAILABLE = False
USE_LIBEDIT = False
+
from docutils.utils import column_width
+
import sphinx.locale
from sphinx import __display_version__, package_dir
from sphinx.locale import __
from sphinx.util.console import bold, color_terminal, colorize, nocolor, red
from sphinx.util.osutil import ensuredir
from sphinx.util.template import SphinxRenderer
+
if TYPE_CHECKING:
from collections.abc import Callable, Sequence
-EXTENSIONS = {'autodoc': __('automatically insert docstrings from modules'),
+
+EXTENSIONS = {
+ 'autodoc': __('automatically insert docstrings from modules'),
'doctest': __('automatically test code snippets in doctest blocks'),
- 'intersphinx': __(
- 'link between Sphinx documentation of different projects'), 'todo': __(
- 'write "todo" entries that can be shown or hidden on build'),
- 'coverage': __('checks for documentation coverage'), 'imgmath': __(
- 'include math, rendered as PNG or SVG images'), 'mathjax': __(
- 'include math, rendered in the browser by MathJax'), 'ifconfig': __(
- 'conditional inclusion of content based on config values'), 'viewcode':
- __('include links to the source code of documented Python objects'),
- 'githubpages': __(
- 'create .nojekyll file to publish the document on GitHub pages')}
-DEFAULTS = {'path': '.', 'sep': False, 'dot': '_', 'language': None,
- 'suffix': '.rst', 'master': 'index', 'makefile': True, 'batchfile': True}
+ 'intersphinx': __('link between Sphinx documentation of different projects'),
+ 'todo': __('write "todo" entries that can be shown or hidden on build'),
+ 'coverage': __('checks for documentation coverage'),
+ 'imgmath': __('include math, rendered as PNG or SVG images'),
+ 'mathjax': __('include math, rendered in the browser by MathJax'),
+ 'ifconfig': __('conditional inclusion of content based on config values'),
+ 'viewcode': __('include links to the source code of documented Python objects'),
+ 'githubpages': __('create .nojekyll file to publish the document on GitHub pages'),
+}
+
+DEFAULTS = {
+ 'path': '.',
+ 'sep': False,
+ 'dot': '_',
+ 'language': None,
+ 'suffix': '.rst',
+ 'master': 'index',
+ 'makefile': True,
+ 'batchfile': True,
+}
+
PROMPT_PREFIX = '> '
+
if sys.platform == 'win32':
+ # On Windows, show questions as bold because of color scheme of PowerShell (refs: #5294).
COLOR_QUESTION = 'bold'
else:
COLOR_QUESTION = 'purple'
+# function to get input from terminal -- overridden by the test suite
+def term_input(prompt: str) -> str:
+ if sys.platform == 'win32':
+ # Important: On windows, readline is not enabled by default. In these
+ # environment, escape sequences have been broken. To avoid the
+ # problem, quickstart uses ``print()`` to show prompt.
+ print(prompt, end='')
+ return input('')
+ else:
+ return input(prompt)
+
+
class ValidationError(Exception):
"""Raised for validation errors."""
-class QuickstartRenderer(SphinxRenderer):
+def is_path(x: str) -> str:
+ x = path.expanduser(x)
+ if not path.isdir(x):
+ raise ValidationError(__("Please enter a valid path name."))
+ return x
+
+
+def is_path_or_empty(x: str) -> str:
+ if x == '':
+ return x
+ return is_path(x)
+
+
+def allow_empty(x: str) -> str:
+ return x
+
+
+def nonempty(x: str) -> str:
+ if not x:
+ raise ValidationError(__("Please enter some text."))
+ return x
+
+
+def choice(*l: str) -> Callable[[str], str]:
+ def val(x: str) -> str:
+ if x not in l:
+ raise ValidationError(__('Please enter one of %s.') % ', '.join(l))
+ return x
+ return val
+
- def __init__(self, templatedir: str='') ->None:
+def boolean(x: str) -> bool:
+ if x.upper() not in ('Y', 'YES', 'N', 'NO'):
+ raise ValidationError(__("Please enter either 'y' or 'n'."))
+ return x.upper() in ('Y', 'YES')
+
+
+def suffix(x: str) -> str:
+ if not (x[0:1] == '.' and len(x) > 1):
+ raise ValidationError(__("Please enter a file suffix, e.g. '.rst' or '.txt'."))
+ return x
+
+
+def ok(x: str) -> str:
+ return x
+
+
+def do_prompt(
+ text: str, default: str | None = None, validator: Callable[[str], Any] = nonempty,
+) -> str | bool:
+ while True:
+ if default is not None:
+ prompt = PROMPT_PREFIX + f'{text} [{default}]: '
+ else:
+ prompt = PROMPT_PREFIX + text + ': '
+ if USE_LIBEDIT:
+ # Note: libedit has a problem for combination of ``input()`` and escape
+ # sequence (see #5335). To avoid the problem, all prompts are not colored
+ # on libedit.
+ pass
+ elif READLINE_AVAILABLE:
+ # pass input_mode=True if readline available
+ prompt = colorize(COLOR_QUESTION, prompt, input_mode=True)
+ else:
+ prompt = colorize(COLOR_QUESTION, prompt, input_mode=False)
+ x = term_input(prompt).strip()
+ if default and not x:
+ x = default
+ try:
+ x = validator(x)
+ except ValidationError as err:
+ print(red('* ' + str(err)))
+ continue
+ break
+ return x
+
+
+class QuickstartRenderer(SphinxRenderer):
+ def __init__(self, templatedir: str = '') -> None:
self.templatedir = templatedir
super().__init__()
- def _has_custom_template(self, template_name: str) ->bool:
+ def _has_custom_template(self, template_name: str) -> bool:
"""Check if custom template file exists.
Note: Please don't use this function from extensions.
It will be removed in the future without deprecation period.
"""
- pass
+ template = path.join(self.templatedir, path.basename(template_name))
+ return bool(self.templatedir) and path.exists(template)
+ def render(self, template_name: str, context: dict[str, Any]) -> str:
+ if self._has_custom_template(template_name):
+ custom_template = path.join(self.templatedir, path.basename(template_name))
+ return self.render_from_file(custom_template, context)
+ else:
+ return super().render(template_name, context)
-def ask_user(d: dict[str, Any]) ->None:
+
+def ask_user(d: dict[str, Any]) -> None:
"""Ask the user for quickstart values missing from *d*.
Values are:
@@ -89,13 +205,399 @@ def ask_user(d: dict[str, Any]) ->None:
* makefile: make Makefile
* batchfile: make command file
"""
- pass
+ print(bold(__('Welcome to the Sphinx %s quickstart utility.')) % __display_version__)
+ print()
+ print(__('Please enter values for the following settings (just press Enter to\n'
+ 'accept a default value, if one is given in brackets).'))
+
+ if 'path' in d:
+ print()
+ print(bold(__('Selected root path: %s')) % d['path'])
+ else:
+ print()
+ print(__('Enter the root path for documentation.'))
+ d['path'] = do_prompt(__('Root path for the documentation'), '.', is_path)
+
+ while path.isfile(path.join(d['path'], 'conf.py')) or \
+ path.isfile(path.join(d['path'], 'source', 'conf.py')):
+ print()
+ print(bold(__('Error: an existing conf.py has been found in the '
+ 'selected root path.')))
+ print(__('sphinx-quickstart will not overwrite existing Sphinx projects.'))
+ print()
+ d['path'] = do_prompt(__('Please enter a new root path (or just Enter to exit)'),
+ '', is_path_or_empty)
+ if not d['path']:
+ raise SystemExit(1)
+
+ if 'sep' not in d:
+ print()
+ print(__('You have two options for placing the build directory for Sphinx output.\n'
+ 'Either, you use a directory "_build" within the root path, or you separate\n'
+ '"source" and "build" directories within the root path.'))
+ d['sep'] = do_prompt(__('Separate source and build directories (y/n)'), 'n', boolean)
+
+ if 'dot' not in d:
+ print()
+ print(__('Inside the root directory, two more directories will be created; "_templates"\n' # NoQA: E501
+ 'for custom HTML templates and "_static" for custom stylesheets and other static\n' # NoQA: E501
+ 'files. You can enter another prefix (such as ".") to replace the underscore.')) # NoQA: E501
+ d['dot'] = do_prompt(__('Name prefix for templates and static dir'), '_', ok)
+
+ if 'project' not in d:
+ print()
+ print(__('The project name will occur in several places in the built documentation.'))
+ d['project'] = do_prompt(__('Project name'))
+ if 'author' not in d:
+ d['author'] = do_prompt(__('Author name(s)'))
+
+ if 'version' not in d:
+ print()
+ print(__('Sphinx has the notion of a "version" and a "release" for the\n'
+ 'software. Each version can have multiple releases. For example, for\n'
+ 'Python the version is something like 2.5 or 3.0, while the release is\n'
+ "something like 2.5.1 or 3.0a1. If you don't need this dual structure,\n"
+ 'just set both to the same value.'))
+ d['version'] = do_prompt(__('Project version'), '', allow_empty)
+ if 'release' not in d:
+ d['release'] = do_prompt(__('Project release'), d['version'], allow_empty)
+
+ if 'language' not in d:
+ print()
+ print(__(
+ 'If the documents are to be written in a language other than English,\n'
+ 'you can select a language here by its language code. Sphinx will then\n'
+ 'translate text that it generates into that language.\n'
+ '\n'
+ 'For a list of supported codes, see\n'
+ 'https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-language.',
+ ))
+ d['language'] = do_prompt(__('Project language'), 'en')
+ if d['language'] == 'en':
+ d['language'] = None
+
+ if 'suffix' not in d:
+ print()
+ print(__('The file name suffix for source files. Commonly, this is either ".txt"\n'
+ 'or ".rst". Only files with this suffix are considered documents.'))
+ d['suffix'] = do_prompt(__('Source file suffix'), '.rst', suffix)
+
+ if 'master' not in d:
+ print()
+ print(__('One document is special in that it is considered the top node of the\n'
+ '"contents tree", that is, it is the root of the hierarchical structure\n'
+ 'of the documents. Normally, this is "index", but if your "index"\n'
+ 'document is a custom template, you can also set this to another filename.'))
+ d['master'] = do_prompt(__('Name of your master document (without suffix)'), 'index')
+
+ while path.isfile(path.join(d['path'], d['master'] + d['suffix'])) or \
+ path.isfile(path.join(d['path'], 'source', d['master'] + d['suffix'])):
+ print()
+ print(bold(__('Error: the master file %s has already been found in the '
+ 'selected root path.') % (d['master'] + d['suffix'])))
+ print(__('sphinx-quickstart will not overwrite the existing file.'))
+ print()
+ d['master'] = do_prompt(__('Please enter a new file name, or rename the '
+ 'existing file and press Enter'), d['master'])
+
+ if 'extensions' not in d:
+ print(__('Indicate which of the following Sphinx extensions should be enabled:'))
+ d['extensions'] = []
+ for name, description in EXTENSIONS.items():
+ if do_prompt(f'{name}: {description} (y/n)', 'n', boolean):
+ d['extensions'].append('sphinx.ext.%s' % name)
+ # Handle conflicting options
+ if {'sphinx.ext.imgmath', 'sphinx.ext.mathjax'}.issubset(d['extensions']):
+ print(__('Note: imgmath and mathjax cannot be enabled at the same time. '
+ 'imgmath has been deselected.'))
+ d['extensions'].remove('sphinx.ext.imgmath')
-def generate(d: dict[str, Any], overwrite: bool=True, silent: bool=False,
- templatedir: (str | None)=None) ->None:
+ if 'makefile' not in d:
+ print()
+ print(__('A Makefile and a Windows command file can be generated for you so that you\n'
+ "only have to run e.g. `make html' instead of invoking sphinx-build\n"
+ 'directly.'))
+ d['makefile'] = do_prompt(__('Create Makefile? (y/n)'), 'y', boolean)
+
+ if 'batchfile' not in d:
+ d['batchfile'] = do_prompt(__('Create Windows command file? (y/n)'), 'y', boolean)
+ print()
+
+
+def generate(
+ d: dict[str, Any],
+ overwrite: bool = True,
+ silent: bool = False,
+ templatedir: str | None = None,
+) -> None:
"""Generate project based on values in *d*."""
- pass
+ template = QuickstartRenderer(templatedir or '')
+
+ if 'mastertoctree' not in d:
+ d['mastertoctree'] = ''
+ if 'mastertocmaxdepth' not in d:
+ d['mastertocmaxdepth'] = 2
+
+ d['root_doc'] = d['master']
+ d['now'] = time.asctime()
+ d['project_underline'] = column_width(d['project']) * '='
+ d.setdefault('extensions', [])
+ d['copyright'] = time.strftime('%Y') + ', ' + d['author']
+
+ d["path"] = os.path.abspath(d['path'])
+ ensuredir(d['path'])
+
+ srcdir = path.join(d['path'], 'source') if d['sep'] else d['path']
+
+ ensuredir(srcdir)
+ if d['sep']:
+ builddir = path.join(d['path'], 'build')
+ d['exclude_patterns'] = ''
+ else:
+ builddir = path.join(srcdir, d['dot'] + 'build')
+ exclude_patterns = map(repr, [
+ d['dot'] + 'build',
+ 'Thumbs.db', '.DS_Store',
+ ])
+ d['exclude_patterns'] = ', '.join(exclude_patterns)
+ ensuredir(builddir)
+ ensuredir(path.join(srcdir, d['dot'] + 'templates'))
+ ensuredir(path.join(srcdir, d['dot'] + 'static'))
+
+ def write_file(fpath: str, content: str, newline: str | None = None) -> None:
+ if overwrite or not path.isfile(fpath):
+ if 'quiet' not in d:
+ print(__('Creating file %s.') % fpath)
+ with open(fpath, 'w', encoding='utf-8', newline=newline) as f:
+ f.write(content)
+ else:
+ if 'quiet' not in d:
+ print(__('File %s already exists, skipping.') % fpath)
+
+ conf_path = os.path.join(templatedir, 'conf.py.jinja') if templatedir else None
+ if not conf_path or not path.isfile(conf_path):
+ conf_path = os.path.join(package_dir, 'templates', 'quickstart', 'conf.py.jinja')
+ with open(conf_path, encoding="utf-8") as f:
+ conf_text = f.read()
+
+ write_file(path.join(srcdir, 'conf.py'), template.render_string(conf_text, d))
+
+ masterfile = path.join(srcdir, d['master'] + d['suffix'])
+ if template._has_custom_template('quickstart/master_doc.rst.jinja'):
+ msg = ('A custom template `master_doc.rst.jinja` found. It has been renamed to '
+ '`root_doc.rst.jinja`. Please rename it on your project too.')
+ print(colorize('red', msg))
+ write_file(masterfile, template.render('quickstart/master_doc.rst.jinja', d))
+ else:
+ write_file(masterfile, template.render('quickstart/root_doc.rst.jinja', d))
+
+ makefile_template = 'quickstart/Makefile.new.jinja'
+ batchfile_template = 'quickstart/make.bat.new.jinja'
+
+ if d['makefile'] is True:
+ d['rsrcdir'] = 'source' if d['sep'] else '.'
+ d['rbuilddir'] = 'build' if d['sep'] else d['dot'] + 'build'
+ # use binary mode, to avoid writing \r\n on Windows
+ write_file(path.join(d['path'], 'Makefile'),
+ template.render(makefile_template, d), '\n')
+
+ if d['batchfile'] is True:
+ d['rsrcdir'] = 'source' if d['sep'] else '.'
+ d['rbuilddir'] = 'build' if d['sep'] else d['dot'] + 'build'
+ write_file(path.join(d['path'], 'make.bat'),
+ template.render(batchfile_template, d), '\r\n')
+
+ if silent:
+ return
+ print()
+ print(bold(__('Finished: An initial directory structure has been created.')))
+ print()
+ print(__('You should now populate your master file %s and create other documentation\n'
+ 'source files. ') % masterfile, end='')
+ if d['makefile'] or d['batchfile']:
+ print(__('Use the Makefile to build the docs, like so:\n'
+ ' make builder'))
+ else:
+ print(__('Use the sphinx-build command to build the docs, like so:\n'
+ ' sphinx-build -b builder %s %s') % (srcdir, builddir))
+ print(__('where "builder" is one of the supported builders, '
+ 'e.g. html, latex or linkcheck.'))
+ print()
+
+
+def valid_dir(d: dict[str, Any]) -> bool:
+ dir = d['path']
+ if not path.exists(dir):
+ return True
+ if not path.isdir(dir):
+ return False
+
+ if {'Makefile', 'make.bat'} & set(os.listdir(dir)):
+ return False
+
+ if d['sep']:
+ dir = os.path.join('source', dir)
+ if not path.exists(dir):
+ return True
+ if not path.isdir(dir):
+ return False
+
+ reserved_names = [
+ 'conf.py',
+ d['dot'] + 'static',
+ d['dot'] + 'templates',
+ d['master'] + d['suffix'],
+ ]
+ return not set(reserved_names) & set(os.listdir(dir))
+
+
+def get_parser() -> argparse.ArgumentParser:
+ description = __(
+ "\n"
+ "Generate required files for a Sphinx project.\n"
+ "\n"
+ "sphinx-quickstart is an interactive tool that asks some questions about your\n"
+ "project and then generates a complete documentation directory and sample\n"
+ "Makefile to be used with sphinx-build.\n",
+ )
+ parser = argparse.ArgumentParser(
+ usage='%(prog)s [OPTIONS] <PROJECT_DIR>',
+ epilog=__("For more information, visit <https://www.sphinx-doc.org/>."),
+ description=description)
+
+ parser.add_argument('-q', '--quiet', action='store_true', dest='quiet',
+ default=None,
+ help=__('quiet mode'))
+ parser.add_argument('--version', action='version', dest='show_version',
+ version='%%(prog)s %s' % __display_version__)
+
+ parser.add_argument('path', metavar='PROJECT_DIR', default='.', nargs='?',
+ help=__('project root'))
+
+ group = parser.add_argument_group(__('Structure options'))
+ group.add_argument('--sep', action='store_true', dest='sep', default=None,
+ help=__('if specified, separate source and build dirs'))
+ group.add_argument('--no-sep', action='store_false', dest='sep',
+ help=__('if specified, create build dir under source dir'))
+ group.add_argument('--dot', metavar='DOT', default='_',
+ help=__('replacement for dot in _templates etc.'))
+
+ group = parser.add_argument_group(__('Project basic options'))
+ group.add_argument('-p', '--project', metavar='PROJECT', dest='project',
+ help=__('project name'))
+ group.add_argument('-a', '--author', metavar='AUTHOR', dest='author',
+ help=__('author names'))
+ group.add_argument('-v', metavar='VERSION', dest='version', default='',
+ help=__('version of project'))
+ group.add_argument('-r', '--release', metavar='RELEASE', dest='release',
+ help=__('release of project'))
+ group.add_argument('-l', '--language', metavar='LANGUAGE', dest='language',
+ help=__('document language'))
+ group.add_argument('--suffix', metavar='SUFFIX', default='.rst',
+ help=__('source file suffix'))
+ group.add_argument('--master', metavar='MASTER', default='index',
+ help=__('master document name'))
+ group.add_argument('--epub', action='store_true', default=False,
+ help=__('use epub'))
+
+ group = parser.add_argument_group(__('Extension options'))
+ for ext in EXTENSIONS:
+ group.add_argument('--ext-%s' % ext, action='append_const',
+ const='sphinx.ext.%s' % ext, dest='extensions',
+ help=__('enable %s extension') % ext)
+ group.add_argument('--extensions', metavar='EXTENSIONS', dest='extensions',
+ action='append', help=__('enable arbitrary extensions'))
+
+ group = parser.add_argument_group(__('Makefile and Batchfile creation'))
+ group.add_argument('--makefile', action='store_true', dest='makefile', default=True,
+ help=__('create makefile'))
+ group.add_argument('--no-makefile', action='store_false', dest='makefile',
+ help=__('do not create makefile'))
+ group.add_argument('--batchfile', action='store_true', dest='batchfile', default=True,
+ help=__('create batchfile'))
+ group.add_argument('--no-batchfile', action='store_false',
+ dest='batchfile',
+ help=__('do not create batchfile'))
+ # --use-make-mode is a no-op from Sphinx 8.
+ group.add_argument('-m', '--use-make-mode', action='store_true',
+ dest='make_mode', default=True,
+ help=__('use make-mode for Makefile/make.bat'))
+
+ group = parser.add_argument_group(__('Project templating'))
+ group.add_argument('-t', '--templatedir', metavar='TEMPLATEDIR',
+ dest='templatedir',
+ help=__('template directory for template files'))
+ group.add_argument('-d', metavar='NAME=VALUE', action='append',
+ dest='variables',
+ help=__('define a template variable'))
+
+ return parser
+
+
+def main(argv: Sequence[str] = (), /) -> int:
+ locale.setlocale(locale.LC_ALL, '')
+ sphinx.locale.init_console()
+
+ if not color_terminal():
+ nocolor()
+
+ # parse options
+ parser = get_parser()
+ try:
+ args = parser.parse_args(argv or sys.argv[1:])
+ except SystemExit as err:
+ return err.code # type: ignore[return-value]
+
+ d = vars(args)
+ # delete None or False value
+ d = {k: v for k, v in d.items() if v is not None}
+
+ # handle use of CSV-style extension values
+ d.setdefault('extensions', [])
+ for ext in d['extensions'][:]:
+ if ',' in ext:
+ d['extensions'].remove(ext)
+ d['extensions'].extend(ext.split(','))
+
+ try:
+ if 'quiet' in d:
+ if not {'project', 'author'}.issubset(d):
+ print(__('"quiet" is specified, but any of "project" or '
+ '"author" is not specified.'))
+ return 1
+
+ if {'quiet', 'project', 'author'}.issubset(d):
+ # quiet mode with all required params satisfied, use default
+ d.setdefault('version', '')
+ d.setdefault('release', d['version'])
+ d2 = DEFAULTS.copy()
+ d2.update(d)
+ d = d2
+
+ if not valid_dir(d):
+ print()
+ print(bold(__('Error: specified path is not a directory, or sphinx'
+ ' files already exist.')))
+ print(__('sphinx-quickstart only generate into a empty directory.'
+ ' Please specify a new root path.'))
+ return 1
+ else:
+ ask_user(d)
+ except (KeyboardInterrupt, EOFError):
+ print()
+ print('[Interrupted.]')
+ return 130 # 128 + SIGINT
+
+ for variable in d.get('variables', []):
+ try:
+ name, value = variable.split('=')
+ d[name] = value
+ except ValueError:
+ print(__('Invalid template variable: %s') % variable)
+
+ generate(d, overwrite=False, templatedir=args.templatedir)
+ return 0
if __name__ == '__main__':
diff --git a/sphinx/config.py b/sphinx/config.py
index 035734a48..bae92140d 100644
--- a/sphinx/config.py
+++ b/sphinx/config.py
@@ -1,5 +1,7 @@
"""Build configuration file handling."""
+
from __future__ import annotations
+
import sys
import time
import traceback
@@ -7,28 +9,40 @@ import types
import warnings
from os import getenv, path
from typing import TYPE_CHECKING, Any, Literal, NamedTuple
+
from sphinx.deprecation import RemovedInSphinx90Warning
from sphinx.errors import ConfigError, ExtensionError
from sphinx.locale import _, __
from sphinx.util import logging
from sphinx.util.osutil import fs_encoding
+
if sys.version_info >= (3, 11):
from contextlib import chdir
else:
from sphinx.util.osutil import _chdir as chdir
+
if TYPE_CHECKING:
import os
from collections.abc import Collection, Iterable, Iterator, Sequence, Set
from typing import TypeAlias
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.tags import Tags
from sphinx.util.typing import ExtensionMetadata, _ExtensionSetupFunc
+
logger = logging.getLogger(__name__)
-_ConfigRebuild: TypeAlias = Literal['', 'env', 'epub', 'gettext', 'html',
- 'applehelp', 'devhelp']
+
+_ConfigRebuild: TypeAlias = Literal[
+ '', 'env', 'epub', 'gettext', 'html',
+ # sphinxcontrib-applehelp
+ 'applehelp',
+ # sphinxcontrib-devhelp
+ 'devhelp',
+]
+
CONFIG_FILENAME = 'conf.py'
-UNSERIALIZABLE_TYPES = type, types.ModuleType, types.FunctionType
+UNSERIALIZABLE_TYPES = (type, types.ModuleType, types.FunctionType)
class ConfigValue(NamedTuple):
@@ -37,9 +51,29 @@ class ConfigValue(NamedTuple):
rebuild: _ConfigRebuild
-def is_serializable(obj: object, *, _seen: frozenset[int]=frozenset()) ->bool:
+def is_serializable(obj: object, *, _seen: frozenset[int] = frozenset()) -> bool:
"""Check if an object is serializable or not."""
- pass
+ if isinstance(obj, UNSERIALIZABLE_TYPES):
+ return False
+
+ # use id() to handle un-hashable objects
+ if id(obj) in _seen:
+ return True
+
+ if isinstance(obj, dict):
+ seen = _seen | {id(obj)}
+ return all(
+ is_serializable(key, _seen=seen) and is_serializable(value, _seen=seen)
+ for key, value in obj.items()
+ )
+ elif isinstance(obj, list | tuple | set | frozenset):
+ seen = _seen | {id(obj)}
+ return all(is_serializable(item, _seen=seen) for item in obj)
+
+ # if an issue occurs for a non-serializable type, pickle will complain
+ # since the object is likely coming from a third-party extension
+ # (we natively expect 'simple' types and not weird ones)
+ return True
class ENUM:
@@ -49,23 +83,34 @@ class ENUM:
app.add_config_value('latex_show_urls', 'no', None, ENUM('no', 'footnote', 'inline'))
"""
- def __init__(self, *candidates: (str | bool | None)) ->None:
+ def __init__(self, *candidates: str | bool | None) -> None:
self.candidates = candidates
+ def match(self, value: str | list | tuple) -> bool:
+ if isinstance(value, list | tuple):
+ return all(item in self.candidates for item in value)
+ else:
+ return value in self.candidates
+
-_OptValidTypes: TypeAlias = tuple[()] | tuple[type, ...] | frozenset[type
- ] | ENUM
+_OptValidTypes: TypeAlias = tuple[()] | tuple[type, ...] | frozenset[type] | ENUM
class _Opt:
__slots__ = 'default', 'rebuild', 'valid_types', 'description'
+
default: Any
rebuild: _ConfigRebuild
valid_types: _OptValidTypes
description: str
- def __init__(self, default: Any, rebuild: _ConfigRebuild, valid_types:
- _OptValidTypes, description: str='') ->None:
+ def __init__(
+ self,
+ default: Any,
+ rebuild: _ConfigRebuild,
+ valid_types: _OptValidTypes,
+ description: str = '',
+ ) -> None:
"""Configuration option type for Sphinx.
The type is intended to be immutable; changing the field values
@@ -79,144 +124,171 @@ class _Opt:
super().__setattr__('valid_types', valid_types)
super().__setattr__('description', description)
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return (
- f'{self.__class__.__qualname__}(default={self.default!r}, rebuild={self.rebuild!r}, valid_types={self.rebuild!r}, description={self.description!r})'
- )
-
- def __eq__(self, other: object) ->bool:
+ f'{self.__class__.__qualname__}('
+ f'default={self.default!r}, '
+ f'rebuild={self.rebuild!r}, '
+ f'valid_types={self.rebuild!r}, '
+ f'description={self.description!r})'
+ )
+
+ def __eq__(self, other: object) -> bool:
if isinstance(other, _Opt):
- self_tpl = (self.default, self.rebuild, self.valid_types, self.
- description)
- other_tpl = (other.default, other.rebuild, other.valid_types,
- self.description)
+ self_tpl = (self.default, self.rebuild, self.valid_types, self.description)
+ other_tpl = (other.default, other.rebuild, other.valid_types, self.description)
return self_tpl == other_tpl
return NotImplemented
- def __lt__(self, other: _Opt) ->bool:
+ def __lt__(self, other: _Opt) -> bool:
if self.__class__ is other.__class__:
- self_tpl = (self.default, self.rebuild, self.valid_types, self.
- description)
- other_tpl = (other.default, other.rebuild, other.valid_types,
- self.description)
+ self_tpl = (self.default, self.rebuild, self.valid_types, self.description)
+ other_tpl = (other.default, other.rebuild, other.valid_types, self.description)
return self_tpl > other_tpl
return NotImplemented
- def __hash__(self) ->int:
- return hash((self.default, self.rebuild, self.valid_types, self.
- description))
+ def __hash__(self) -> int:
+ return hash((self.default, self.rebuild, self.valid_types, self.description))
- def __setattr__(self, key: str, value: Any) ->None:
+ def __setattr__(self, key: str, value: Any) -> None:
if key in {'default', 'rebuild', 'valid_types', 'description'}:
- msg = (
- f'{self.__class__.__name__!r} object does not support assignment to {key!r}'
- )
+ msg = f'{self.__class__.__name__!r} object does not support assignment to {key!r}'
raise TypeError(msg)
super().__setattr__(key, value)
- def __delattr__(self, key: str) ->None:
+ def __delattr__(self, key: str) -> None:
if key in {'default', 'rebuild', 'valid_types', 'description'}:
- msg = (
- f'{self.__class__.__name__!r} object does not support deletion of {key!r}'
- )
+ msg = f'{self.__class__.__name__!r} object does not support deletion of {key!r}'
raise TypeError(msg)
super().__delattr__(key)
- def __getstate__(self) ->tuple[Any, _ConfigRebuild, _OptValidTypes, str]:
+ def __getstate__(self) -> tuple[Any, _ConfigRebuild, _OptValidTypes, str]:
return self.default, self.rebuild, self.valid_types, self.description
- def __setstate__(self, state: tuple[Any, _ConfigRebuild, _OptValidTypes,
- str]) ->None:
+ def __setstate__(
+ self, state: tuple[Any, _ConfigRebuild, _OptValidTypes, str]) -> None:
default, rebuild, valid_types, description = state
super().__setattr__('default', default)
super().__setattr__('rebuild', rebuild)
super().__setattr__('valid_types', valid_types)
super().__setattr__('description', description)
- def __getitem__(self, item: (int | slice)) ->Any:
+ def __getitem__(self, item: int | slice) -> Any:
warnings.warn(
- f"The {self.__class__.__name__!r} object tuple interface is deprecated, use attribute access instead for 'default', 'rebuild', and 'valid_types'."
- , RemovedInSphinx90Warning, stacklevel=2)
+ f'The {self.__class__.__name__!r} object tuple interface is deprecated, '
+ "use attribute access instead for 'default', 'rebuild', and 'valid_types'.",
+ RemovedInSphinx90Warning, stacklevel=2)
return (self.default, self.rebuild, self.valid_types)[item]
class Config:
- """Configuration file abstraction.
+ r"""Configuration file abstraction.
The Config object makes the values of all config options available as
attributes.
- It is exposed via the :py:class:`~sphinx.application.Sphinx`\\ ``.config``
- and :py:class:`sphinx.environment.BuildEnvironment`\\ ``.config`` attributes.
+ It is exposed via the :py:class:`~sphinx.application.Sphinx`\ ``.config``
+ and :py:class:`sphinx.environment.BuildEnvironment`\ ``.config`` attributes.
For example, to get the value of :confval:`language`, use either
``app.config.language`` or ``env.config.language``.
"""
- config_values: dict[str, _Opt] = {'project': _Opt(
- 'Project name not set', 'env', ()), 'author': _Opt(
- 'Author name not set', 'env', ()), 'project_copyright': _Opt('',
- 'html', frozenset((str, tuple, list))), 'copyright': _Opt(lambda
- config: config.project_copyright, 'html', frozenset((str, tuple,
- list))), 'version': _Opt('', 'env', ()), 'release': _Opt('', 'env',
- ()), 'today': _Opt('', 'env', ()), 'today_fmt': _Opt(None, 'env',
- frozenset((str,))), 'language': _Opt('en', 'env', frozenset((str,))
- ), 'locale_dirs': _Opt(['locales'], 'env', ()),
- 'figure_language_filename': _Opt('{root}.{language}{ext}', 'env',
- frozenset((str,))), 'gettext_allow_fuzzy_translations': _Opt(False,
- 'gettext', ()), 'translation_progress_classes': _Opt(False, 'env',
- ENUM(True, False, 'translated', 'untranslated')), 'master_doc':
- _Opt('index', 'env', ()), 'root_doc': _Opt(lambda config: config.
- master_doc, 'env', ()), 'source_suffix': _Opt({'.rst':
- 'restructuredtext'}, 'env', Any), 'source_encoding': _Opt(
- 'utf-8-sig', 'env', ()), 'exclude_patterns': _Opt([], 'env',
- frozenset((str,))), 'include_patterns': _Opt(['**'], 'env',
- frozenset((str,))), 'default_role': _Opt(None, 'env', frozenset((
- str,))), 'add_function_parentheses': _Opt(True, 'env', ()),
- 'add_module_names': _Opt(True, 'env', ()), 'toc_object_entries':
- _Opt(True, 'env', frozenset((bool,))),
- 'toc_object_entries_show_parents': _Opt('domain', 'env', ENUM(
- 'domain', 'all', 'hide')), 'trim_footnote_reference_space': _Opt(
- False, 'env', ()), 'show_authors': _Opt(False, 'env', ()),
+
+ # The values are:
+ # 1. Default
+ # 2. What needs to be rebuilt if changed
+ # 3. Valid types
+
+ # If you add a value here, remember to include it in the docs!
+
+ config_values: dict[str, _Opt] = {
+ # general options
+ 'project': _Opt('Project name not set', 'env', ()),
+ 'author': _Opt('Author name not set', 'env', ()),
+ 'project_copyright': _Opt('', 'html', frozenset((str, tuple, list))),
+ 'copyright': _Opt(
+ lambda config: config.project_copyright, 'html', frozenset((str, tuple, list))),
+ 'version': _Opt('', 'env', ()),
+ 'release': _Opt('', 'env', ()),
+ 'today': _Opt('', 'env', ()),
+ # the real default is locale-dependent
+ 'today_fmt': _Opt(None, 'env', frozenset((str,))),
+
+ 'language': _Opt('en', 'env', frozenset((str,))),
+ 'locale_dirs': _Opt(['locales'], 'env', ()),
+ 'figure_language_filename': _Opt('{root}.{language}{ext}', 'env', frozenset((str,))),
+ 'gettext_allow_fuzzy_translations': _Opt(False, 'gettext', ()),
+ 'translation_progress_classes': _Opt(
+ False, 'env', ENUM(True, False, 'translated', 'untranslated')),
+
+ 'master_doc': _Opt('index', 'env', ()),
+ 'root_doc': _Opt(lambda config: config.master_doc, 'env', ()),
+ # ``source_suffix`` type is actually ``dict[str, str | None]``:
+ # see ``convert_source_suffix()`` below.
+ 'source_suffix': _Opt(
+ {'.rst': 'restructuredtext'}, 'env', Any), # type: ignore[arg-type]
+ 'source_encoding': _Opt('utf-8-sig', 'env', ()),
+ 'exclude_patterns': _Opt([], 'env', frozenset((str,))),
+ 'include_patterns': _Opt(["**"], 'env', frozenset((str,))),
+ 'default_role': _Opt(None, 'env', frozenset((str,))),
+ 'add_function_parentheses': _Opt(True, 'env', ()),
+ 'add_module_names': _Opt(True, 'env', ()),
+ 'toc_object_entries': _Opt(True, 'env', frozenset((bool,))),
+ 'toc_object_entries_show_parents': _Opt(
+ 'domain', 'env', ENUM('domain', 'all', 'hide')),
+ 'trim_footnote_reference_space': _Opt(False, 'env', ()),
+ 'show_authors': _Opt(False, 'env', ()),
'pygments_style': _Opt(None, 'html', frozenset((str,))),
'highlight_language': _Opt('default', 'env', ()),
- 'highlight_options': _Opt({}, 'env', ()), 'templates_path': _Opt([],
- 'html', ()), 'template_bridge': _Opt(None, 'html', frozenset((str,)
- )), 'keep_warnings': _Opt(False, 'env', ()), 'suppress_warnings':
- _Opt([], 'env', ()), 'show_warning_types': _Opt(True, 'env',
- frozenset((bool,))), 'modindex_common_prefix': _Opt([], 'html', ()),
- 'rst_epilog': _Opt(None, 'env', frozenset((str,))), 'rst_prolog':
- _Opt(None, 'env', frozenset((str,))), 'trim_doctest_flags': _Opt(
- True, 'env', ()), 'primary_domain': _Opt('py', 'env', frozenset((
- types.NoneType,))), 'needs_sphinx': _Opt(None, '', frozenset((str,)
- )), 'needs_extensions': _Opt({}, '', ()), 'manpages_url': _Opt(None,
- 'env', ()), 'nitpicky': _Opt(False, '', ()), 'nitpick_ignore': _Opt
- ([], '', frozenset((set, list, tuple))), 'nitpick_ignore_regex':
- _Opt([], '', frozenset((set, list, tuple))), 'numfig': _Opt(False,
- 'env', ()), 'numfig_secnum_depth': _Opt(1, 'env', ()),
- 'numfig_format': _Opt({}, 'env', ()),
- 'maximum_signature_line_length': _Opt(None, 'env', frozenset((int,
- types.NoneType))), 'math_number_all': _Opt(False, 'env', ()),
+ 'highlight_options': _Opt({}, 'env', ()),
+ 'templates_path': _Opt([], 'html', ()),
+ 'template_bridge': _Opt(None, 'html', frozenset((str,))),
+ 'keep_warnings': _Opt(False, 'env', ()),
+ 'suppress_warnings': _Opt([], 'env', ()),
+ 'show_warning_types': _Opt(True, 'env', frozenset((bool,))),
+ 'modindex_common_prefix': _Opt([], 'html', ()),
+ 'rst_epilog': _Opt(None, 'env', frozenset((str,))),
+ 'rst_prolog': _Opt(None, 'env', frozenset((str,))),
+ 'trim_doctest_flags': _Opt(True, 'env', ()),
+ 'primary_domain': _Opt('py', 'env', frozenset((types.NoneType,))),
+ 'needs_sphinx': _Opt(None, '', frozenset((str,))),
+ 'needs_extensions': _Opt({}, '', ()),
+ 'manpages_url': _Opt(None, 'env', ()),
+ 'nitpicky': _Opt(False, '', ()),
+ 'nitpick_ignore': _Opt([], '', frozenset((set, list, tuple))),
+ 'nitpick_ignore_regex': _Opt([], '', frozenset((set, list, tuple))),
+ 'numfig': _Opt(False, 'env', ()),
+ 'numfig_secnum_depth': _Opt(1, 'env', ()),
+ 'numfig_format': _Opt({}, 'env', ()), # will be initialized in init_numfig_format()
+ 'maximum_signature_line_length': _Opt(
+ None, 'env', frozenset((int, types.NoneType))),
+ 'math_number_all': _Opt(False, 'env', ()),
'math_eqref_format': _Opt(None, 'env', frozenset((str,))),
- 'math_numfig': _Opt(True, 'env', ()), 'math_numsep': _Opt('.',
- 'env', frozenset((str,))), 'tls_verify': _Opt(True, 'env', ()),
- 'tls_cacerts': _Opt(None, 'env', ()), 'user_agent': _Opt(None,
- 'env', frozenset((str,))), 'smartquotes': _Opt(True, 'env', ()),
+ 'math_numfig': _Opt(True, 'env', ()),
+ 'math_numsep': _Opt('.', 'env', frozenset((str,))),
+ 'tls_verify': _Opt(True, 'env', ()),
+ 'tls_cacerts': _Opt(None, 'env', ()),
+ 'user_agent': _Opt(None, 'env', frozenset((str,))),
+ 'smartquotes': _Opt(True, 'env', ()),
'smartquotes_action': _Opt('qDe', 'env', ()),
- 'smartquotes_excludes': _Opt({'languages': ['ja'], 'builders': [
- 'man', 'text']}, 'env', ()), 'option_emphasise_placeholders': _Opt(
- False, 'env', ())}
+ 'smartquotes_excludes': _Opt(
+ {'languages': ['ja'], 'builders': ['man', 'text']}, 'env', ()),
+ 'option_emphasise_placeholders': _Opt(False, 'env', ()),
+ }
- def __init__(self, config: (dict[str, Any] | None)=None, overrides: (
- dict[str, Any] | None)=None) ->None:
+ def __init__(self, config: dict[str, Any] | None = None,
+ overrides: dict[str, Any] | None = None) -> None:
raw_config: dict[str, Any] = config or {}
self._overrides = dict(overrides) if overrides is not None else {}
self._options = Config.config_values.copy()
self._raw_config = raw_config
+
for name in list(self._overrides.keys()):
if '.' in name:
real_name, key = name.split('.', 1)
- raw_config.setdefault(real_name, {})[key
- ] = self._overrides.pop(name)
+ raw_config.setdefault(real_name, {})[key] = self._overrides.pop(name)
+
self.setup: _ExtensionSetupFunc | None = raw_config.get('setup')
+
if 'extensions' in self._overrides:
extensions = self._overrides.pop('extensions')
if isinstance(extensions, str):
@@ -225,23 +297,102 @@ class Config:
raw_config['extensions'] = extensions
self.extensions: list[str] = raw_config.get('extensions', [])
+ @property
+ def values(self) -> dict[str, _Opt]:
+ return self._options
+
+ @property
+ def overrides(self) -> dict[str, Any]:
+ return self._overrides
+
@classmethod
- def read(cls: type[Config], confdir: (str | os.PathLike[str]),
- overrides: (dict | None)=None, tags: (Tags | None)=None) ->Config:
+ def read(cls: type[Config], confdir: str | os.PathLike[str], overrides: dict | None = None,
+ tags: Tags | None = None) -> Config:
"""Create a Config object from configuration file."""
+ filename = path.join(confdir, CONFIG_FILENAME)
+ if not path.isfile(filename):
+ raise ConfigError(__("config directory doesn't contain a conf.py file (%s)") %
+ confdir)
+ namespace = eval_config_file(filename, tags)
+
+ # Note: Old sphinx projects have been configured as "language = None" because
+ # sphinx-quickstart previously generated this by default.
+ # To keep compatibility, they should be fallback to 'en' for a while
+ # (This conversion should not be removed before 2025-01-01).
+ if namespace.get("language", ...) is None:
+ logger.warning(__("Invalid configuration value found: 'language = None'. "
+ "Update your configuration to a valid language code. "
+ "Falling back to 'en' (English)."))
+ namespace["language"] = "en"
+
+ return cls(namespace, overrides)
+
+ def convert_overrides(self, name: str, value: str) -> Any:
+ opt = self._options[name]
+ default = opt.default
+ valid_types = opt.valid_types
+ if valid_types == Any:
+ return value
+ if (type(default) is bool
+ or (not isinstance(valid_types, ENUM)
+ and len(valid_types) == 1 and bool in valid_types)):
+ if isinstance(valid_types, ENUM) or len(valid_types) > 1:
+ # if valid_types are given, and non-bool valid types exist,
+ # return the value without coercing to a Boolean.
+ return value
+ # given falsy string from a command line option
+ return value not in {'0', ''}
+ if isinstance(default, dict):
+ raise ValueError(__('cannot override dictionary config setting %r, '
+ 'ignoring (use %r to set individual elements)') %
+ (name, f'{name}.key=value'))
+ if isinstance(default, list):
+ return value.split(',')
+ if isinstance(default, int):
+ try:
+ return int(value)
+ except ValueError as exc:
+ raise ValueError(__('invalid number %r for config value %r, ignoring') %
+ (value, name)) from exc
+ if callable(default):
+ return value
+ if isinstance(default, str) or default is None:
+ return value
+ raise ValueError(__('cannot override config setting %r with unsupported '
+ 'type, ignoring') % name)
+
+ @staticmethod
+ def pre_init_values() -> None:
+ # method only retained for compatibility
pass
-
- def __repr__(self) ->str:
+ # warnings.warn(
+ # 'Config.pre_init_values() will be removed in Sphinx 9.0 or later',
+ # RemovedInSphinx90Warning, stacklevel=2)
+
+ def init_values(self) -> None:
+ # method only retained for compatibility
+ self._report_override_warnings()
+ # warnings.warn(
+ # 'Config.init_values() will be removed in Sphinx 9.0 or later',
+ # RemovedInSphinx90Warning, stacklevel=2)
+
+ def _report_override_warnings(self) -> None:
+ for name in self._overrides:
+ if name not in self._options:
+ logger.warning(__('unknown config value %r in override, ignoring'), name)
+
+ def __repr__(self) -> str:
values = []
for opt_name in self._options:
try:
opt_value = getattr(self, opt_name)
except Exception:
opt_value = '<error!>'
- values.append(f'{opt_name}={opt_value!r}')
+ values.append(f"{opt_name}={opt_value!r}")
return self.__class__.__qualname__ + '(' + ', '.join(values) + ')'
- def __setattr__(self, key: str, value: object) ->None:
+ def __setattr__(self, key: str, value: object) -> None:
+ # Ensure aliases update their counterpart.
if key == 'master_doc':
super().__setattr__('root_doc', value)
elif key == 'root_doc':
@@ -252,8 +403,9 @@ class Config:
super().__setattr__('copyright', value)
super().__setattr__(key, value)
- def __getattr__(self, name: str) ->Any:
+ def __getattr__(self, name: str) -> Any:
if name in self._options:
+ # first check command-line overrides
if name in self._overrides:
value = self._overrides[name]
if not isinstance(value, str):
@@ -262,108 +414,233 @@ class Config:
try:
value = self.convert_overrides(name, value)
except ValueError as exc:
- logger.warning('%s', exc)
+ logger.warning("%s", exc)
else:
self.__setattr__(name, value)
return value
+ # then check values from 'conf.py'
if name in self._raw_config:
value = self._raw_config[name]
self.__setattr__(name, value)
return value
+ # finally, fall back to the default value
default = self._options[name].default
if callable(default):
return default(self)
self.__dict__[name] = default
return default
if name.startswith('_'):
- msg = (
- f'{self.__class__.__name__!r} object has no attribute {name!r}'
- )
+ msg = f'{self.__class__.__name__!r} object has no attribute {name!r}'
raise AttributeError(msg)
msg = __('No such config value: %r') % name
raise AttributeError(msg)
- def __getitem__(self, name: str) ->Any:
+ def __getitem__(self, name: str) -> Any:
return getattr(self, name)
- def __setitem__(self, name: str, value: Any) ->None:
+ def __setitem__(self, name: str, value: Any) -> None:
setattr(self, name, value)
- def __delitem__(self, name: str) ->None:
+ def __delitem__(self, name: str) -> None:
delattr(self, name)
- def __contains__(self, name: str) ->bool:
+ def __contains__(self, name: str) -> bool:
return name in self._options
- def __iter__(self) ->Iterator[ConfigValue]:
+ def __iter__(self) -> Iterator[ConfigValue]:
for name, opt in self._options.items():
yield ConfigValue(name, getattr(self, name), opt.rebuild)
- def __getstate__(self) ->dict:
+ def add(self, name: str, default: Any, rebuild: _ConfigRebuild,
+ types: type | Collection[type] | ENUM,
+ description: str = '') -> None:
+ if name in self._options:
+ raise ExtensionError(__('Config value %r already present') % name)
+
+ # standardise rebuild
+ if isinstance(rebuild, bool):
+ rebuild = 'env' if rebuild else ''
+
+ # standardise valid_types
+ valid_types = _validate_valid_types(types)
+ self._options[name] = _Opt(default, rebuild, valid_types, description)
+
+ def filter(self, rebuild: Set[_ConfigRebuild]) -> Iterator[ConfigValue]:
+ if isinstance(rebuild, str):
+ return (value for value in self if value.rebuild == rebuild)
+ return (value for value in self if value.rebuild in rebuild)
+
+ def __getstate__(self) -> dict:
"""Obtains serializable data for pickling."""
- __dict__ = {key: value for key, value in self.__dict__.items() if
- not key.startswith('_') and is_serializable(value)}
+ # remove potentially pickling-problematic values from config
+ __dict__ = {
+ key: value
+ for key, value in self.__dict__.items()
+ if not key.startswith('_') and is_serializable(value)
+ }
+ # create a picklable copy of ``self._options``
__dict__['_options'] = _options = {}
for name, opt in self._options.items():
- if not isinstance(opt, _Opt) and isinstance(opt, tuple) and len(opt
- ) <= 3:
+ if not isinstance(opt, _Opt) and isinstance(opt, tuple) and len(opt) <= 3:
+ # Fix for Furo's ``_update_default``.
self._options[name] = opt = _Opt(*opt)
real_value = getattr(self, name)
if not is_serializable(real_value):
if opt.rebuild:
- logger.warning(__(
- 'cannot cache unpickable configuration value: %r (because it contains a function, class, or module object)'
- ), name, type='config', subtype='cache', once=True)
+ # if the value is not cached, then any build that utilises this cache
+ # will always mark the config value as changed,
+ # and thus always invalidate the cache and perform a rebuild.
+ logger.warning(
+ __('cannot cache unpickable configuration value: %r '
+ '(because it contains a function, class, or module object)'),
+ name,
+ type='config',
+ subtype='cache',
+ once=True,
+ )
+ # omit unserializable value
real_value = None
+ # valid_types is also omitted
_options[name] = real_value, opt.rebuild
+
return __dict__
- def __setstate__(self, state: dict) ->None:
+ def __setstate__(self, state: dict) -> None:
self._overrides = {}
- self._options = {name: _Opt(real_value, rebuild, ()) for name, (
- real_value, rebuild) in state.pop('_options').items()}
+ self._options = {
+ name: _Opt(real_value, rebuild, ())
+ for name, (real_value, rebuild) in state.pop('_options').items()
+ }
self._raw_config = {}
self.__dict__.update(state)
-def eval_config_file(filename: str, tags: (Tags | None)) ->dict[str, Any]:
+def eval_config_file(filename: str, tags: Tags | None) -> dict[str, Any]:
"""Evaluate a config file."""
- pass
-
-
-def convert_source_suffix(app: Sphinx, config: Config) ->None:
+ namespace: dict[str, Any] = {}
+ namespace['__file__'] = filename
+ namespace['tags'] = tags
+
+ with chdir(path.dirname(filename)):
+ # during executing config file, current dir is changed to ``confdir``.
+ try:
+ with open(filename, 'rb') as f:
+ code = compile(f.read(), filename.encode(fs_encoding), 'exec')
+ exec(code, namespace) # NoQA: S102
+ except SyntaxError as err:
+ msg = __("There is a syntax error in your configuration file: %s\n")
+ raise ConfigError(msg % err) from err
+ except SystemExit as exc:
+ msg = __("The configuration file (or one of the modules it imports) "
+ "called sys.exit()")
+ raise ConfigError(msg) from exc
+ except ConfigError:
+ # pass through ConfigError from conf.py as is. It will be shown in console.
+ raise
+ except Exception as exc:
+ msg = __("There is a programmable error in your configuration file:\n\n%s")
+ raise ConfigError(msg % traceback.format_exc()) from exc
+
+ return namespace
+
+
+def _validate_valid_types(
+ valid_types: type | Collection[type] | ENUM, /,
+) -> tuple[()] | tuple[type, ...] | frozenset[type] | ENUM:
+ if not valid_types:
+ return ()
+ if isinstance(valid_types, frozenset | ENUM):
+ return valid_types
+ if isinstance(valid_types, type):
+ return frozenset((valid_types,))
+ if valid_types is Any:
+ return frozenset({Any}) # type: ignore[arg-type]
+ if isinstance(valid_types, set):
+ return frozenset(valid_types)
+ if not isinstance(valid_types, tuple):
+ try:
+ valid_types = tuple(valid_types)
+ except TypeError:
+ logger.warning(__('Failed to convert %r to a set or tuple'), valid_types)
+ return valid_types # type: ignore[return-value]
+ try:
+ return frozenset(valid_types)
+ except TypeError:
+ return valid_types
+
+
+def convert_source_suffix(app: Sphinx, config: Config) -> None:
"""Convert old styled source_suffix to new styled one.
* old style: str or list
* new style: a dict which maps from fileext to filetype
"""
- pass
-
-
-def convert_highlight_options(app: Sphinx, config: Config) ->None:
+ source_suffix = config.source_suffix
+ if isinstance(source_suffix, str):
+ # if str, considers as default filetype (None)
+ #
+ # The default filetype is determined on later step.
+ # By default, it is considered as restructuredtext.
+ config.source_suffix = {source_suffix: 'restructuredtext'}
+ logger.info(__("Converting `source_suffix = %r` to `source_suffix = %r`."),
+ source_suffix, config.source_suffix)
+ elif isinstance(source_suffix, list | tuple):
+ # if list, considers as all of them are default filetype
+ config.source_suffix = dict.fromkeys(source_suffix, 'restructuredtext')
+ logger.info(__("Converting `source_suffix = %r` to `source_suffix = %r`."),
+ source_suffix, config.source_suffix)
+ elif not isinstance(source_suffix, dict):
+ msg = __("The config value `source_suffix' expects a dictionary, "
+ "a string, or a list of strings. Got `%r' instead (type %s).")
+ raise ConfigError(msg % (source_suffix, type(source_suffix)))
+
+
+def convert_highlight_options(app: Sphinx, config: Config) -> None:
"""Convert old styled highlight_options to new styled one.
* old style: options
* new style: a dict which maps from language name to options
"""
- pass
+ options = config.highlight_options
+ if options and not all(isinstance(v, dict) for v in options.values()):
+ # old styled option detected because all values are not dictionary.
+ config.highlight_options = {config.highlight_language: options}
-def init_numfig_format(app: Sphinx, config: Config) ->None:
+def init_numfig_format(app: Sphinx, config: Config) -> None:
"""Initialize :confval:`numfig_format`."""
- pass
+ numfig_format = {'section': _('Section %s'),
+ 'figure': _('Fig. %s'),
+ 'table': _('Table %s'),
+ 'code-block': _('Listing %s')}
+
+ # override default labels by configuration
+ numfig_format.update(config.numfig_format)
+ config.numfig_format = numfig_format
-def correct_copyright_year(_app: Sphinx, config: Config) ->None:
+def correct_copyright_year(_app: Sphinx, config: Config) -> None:
"""Correct values of copyright year that are not coherent with
the SOURCE_DATE_EPOCH environment variable (if set)
See https://reproducible-builds.org/specs/source-date-epoch/
"""
- pass
+ if (source_date_epoch := getenv('SOURCE_DATE_EPOCH')) is None:
+ return
+ source_date_epoch_year = str(time.gmtime(int(source_date_epoch)).tm_year)
-def _substitute_copyright_year(copyright_line: str, replace_year: str) ->str:
+ for k in ('copyright', 'epub_copyright'):
+ if k in config:
+ value: str | Sequence[str] = config[k]
+ if isinstance(value, str):
+ config[k] = _substitute_copyright_year(value, source_date_epoch_year)
+ else:
+ items = (_substitute_copyright_year(x, source_date_epoch_year) for x in value)
+ config[k] = type(value)(items) # type: ignore[call-arg]
+
+
+def _substitute_copyright_year(copyright_line: str, replace_year: str) -> str:
"""Replace the year in a single copyright line.
Legal formats are:
@@ -376,19 +653,118 @@ def _substitute_copyright_year(copyright_line: str, replace_year: str) ->str:
The final year in the string is replaced with ``replace_year``.
"""
- pass
+ if len(copyright_line) < 4 or not copyright_line[:4].isdigit():
+ return copyright_line
+
+ if copyright_line[4:5] in {'', ' ', ','}:
+ return replace_year + copyright_line[4:]
+
+ if copyright_line[4] != '-':
+ return copyright_line
+
+ if copyright_line[5:9].isdigit() and copyright_line[9:10] in {'', ' ', ','}:
+ return copyright_line[:5] + replace_year + copyright_line[9:]
+
+ return copyright_line
-def check_confval_types(app: (Sphinx | None), config: Config) ->None:
+def check_confval_types(app: Sphinx | None, config: Config) -> None:
"""Check all values for deviation from the default value's type, since
that can result in TypeErrors all over the place NB.
"""
- pass
+ for name, opt in config._options.items():
+ default = opt.default
+ valid_types = opt.valid_types
+ value = getattr(config, name)
+
+ if callable(default):
+ default = default(config) # evaluate default value
+ if default is None and not valid_types:
+ continue # neither inferable nor explicitly annotated types
+
+ if valid_types == frozenset({Any}): # any type of value is accepted
+ continue
+
+ if isinstance(valid_types, ENUM):
+ if not valid_types.match(value):
+ msg = __("The config value `{name}` has to be a one of {candidates}, "
+ "but `{current}` is given.")
+ logger.warning(
+ msg.format(name=name, current=value, candidates=valid_types.candidates),
+ once=True,
+ )
+ continue
+
+ type_value = type(value)
+ type_default = type(default)
+
+ if type_value is type_default: # attempt to infer the type
+ continue
+
+ if type_value in valid_types: # check explicitly listed types
+ continue
+
+ common_bases = ({*type_value.__bases__, type_value}
+ & set(type_default.__bases__))
+ common_bases.discard(object)
+ if common_bases:
+ continue # at least we share a non-trivial base class
+
+ if valid_types:
+ msg = __("The config value `{name}' has type `{current.__name__}'; "
+ "expected {permitted}.")
+ wrapped_valid_types = sorted(f"`{c.__name__}'" for c in valid_types)
+ if len(wrapped_valid_types) > 2:
+ permitted = (", ".join(wrapped_valid_types[:-1])
+ + f", or {wrapped_valid_types[-1]}")
+ else:
+ permitted = " or ".join(wrapped_valid_types)
+ logger.warning(
+ msg.format(name=name, current=type_value, permitted=permitted),
+ once=True,
+ )
+ else:
+ msg = __("The config value `{name}' has type `{current.__name__}', "
+ "defaults to `{default.__name__}'.")
+ logger.warning(
+ msg.format(name=name, current=type_value, default=type_default),
+ once=True,
+ )
+
+
+def check_primary_domain(app: Sphinx, config: Config) -> None:
+ primary_domain = config.primary_domain
+ if primary_domain and not app.registry.has_domain(primary_domain):
+ logger.warning(__('primary_domain %r not found, ignored.'), primary_domain)
+ config.primary_domain = None
def check_root_doc(app: Sphinx, env: BuildEnvironment, added: Set[str],
- changed: Set[str], removed: Set[str]) ->Iterable[str]:
+ changed: Set[str], removed: Set[str]) -> Iterable[str]:
"""Adjust root_doc to 'contents' to support an old project which does not have
any root_doc setting.
"""
- pass
+ if (app.config.root_doc == 'index' and
+ 'index' not in app.project.docnames and
+ 'contents' in app.project.docnames):
+ logger.warning(__('Since v2.0, Sphinx uses "index" as root_doc by default. '
+ 'Please add "root_doc = \'contents\'" to your conf.py.'))
+ app.config.root_doc = "contents"
+
+ return changed
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.connect('config-inited', convert_source_suffix, priority=800)
+ app.connect('config-inited', convert_highlight_options, priority=800)
+ app.connect('config-inited', init_numfig_format, priority=800)
+ app.connect('config-inited', correct_copyright_year, priority=800)
+ app.connect('config-inited', check_confval_types, priority=800)
+ app.connect('config-inited', check_primary_domain, priority=800)
+ app.connect('env-get-outdated', check_root_doc)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/deprecation.py b/sphinx/deprecation.py
index ee68dfa87..1baec85bf 100644
--- a/sphinx/deprecation.py
+++ b/sphinx/deprecation.py
@@ -1,5 +1,7 @@
"""Sphinx deprecation classes and utilities."""
+
from __future__ import annotations
+
import warnings
@@ -14,8 +16,14 @@ class RemovedInSphinx10Warning(PendingDeprecationWarning):
RemovedInNextVersionWarning = RemovedInSphinx90Warning
-def _deprecation_warning(module: str, attribute: str, canonical_name: str=
- '', *, remove: tuple[int, int], raises: bool=False) ->None:
+def _deprecation_warning(
+ module: str,
+ attribute: str,
+ canonical_name: str = '',
+ *,
+ remove: tuple[int, int],
+ raises: bool = False,
+) -> None:
"""Helper function for module-level deprecations using ``__getattr__``.
:param module: The module containing a deprecated object.
@@ -51,4 +59,24 @@ def _deprecation_warning(module: str, attribute: str, canonical_name: str=
_deprecation_warning(__name__, name, canonical_name, remove=remove)
return deprecated_object
"""
- pass
+ if remove == (9, 0):
+ warning_class: type[Warning] = RemovedInSphinx90Warning
+ elif remove == (10, 0):
+ warning_class = RemovedInSphinx10Warning
+ else:
+ msg = f'removal version {remove!r} is invalid!'
+ raise RuntimeError(msg)
+
+ qualname = f'{module}.{attribute}'
+ if canonical_name:
+ message = (
+ f'The alias {qualname!r} is deprecated, use {canonical_name!r} instead.'
+ )
+ else:
+ message = f'{qualname!r} is deprecated.'
+
+ if raises:
+ raise AttributeError(message)
+
+ message = f'{message} Check CHANGES for Sphinx API modifications.'
+ warnings.warn(message, warning_class, stacklevel=3)
diff --git a/sphinx/directives/code.py b/sphinx/directives/code.py
index 5e6e9c131..5dc42e5b7 100644
--- a/sphinx/directives/code.py
+++ b/sphinx/directives/code.py
@@ -1,20 +1,26 @@
from __future__ import annotations
+
import sys
import textwrap
from difflib import unified_diff
from typing import TYPE_CHECKING, Any, ClassVar
+
from docutils import nodes
from docutils.parsers.rst import directives
+
from sphinx import addnodes
from sphinx.directives import optional_int
from sphinx.locale import __
from sphinx.util import logging, parselinenos
from sphinx.util.docutils import SphinxDirective
+
if TYPE_CHECKING:
from docutils.nodes import Element, Node
+
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata, OptionSpec
+
logger = logging.getLogger(__name__)
@@ -23,12 +29,64 @@ class Highlight(SphinxDirective):
Directive to set the highlighting language for code blocks, as well
as the threshold for line numbers.
"""
+
has_content = False
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = False
- option_spec: ClassVar[OptionSpec] = {'force': directives.flag,
- 'linenothreshold': directives.positive_int}
+ option_spec: ClassVar[OptionSpec] = {
+ 'force': directives.flag,
+ 'linenothreshold': directives.positive_int,
+ }
+
+ def run(self) -> list[Node]:
+ language = self.arguments[0].strip()
+ linenothreshold = self.options.get('linenothreshold', sys.maxsize)
+ force = 'force' in self.options
+
+ self.env.temp_data['highlight_language'] = language
+ return [addnodes.highlightlang(lang=language,
+ force=force,
+ linenothreshold=linenothreshold)]
+
+
+def dedent_lines(
+ lines: list[str], dedent: int | None, location: tuple[str, int] | None = None,
+) -> list[str]:
+ if dedent is None:
+ return textwrap.dedent(''.join(lines)).splitlines(True)
+
+ if any(s[:dedent].strip() for s in lines):
+ logger.warning(__('non-whitespace stripped by dedent'), location=location)
+
+ new_lines = []
+ for line in lines:
+ new_line = line[dedent:]
+ if line.endswith('\n') and not new_line:
+ new_line = '\n' # keep CRLF
+ new_lines.append(new_line)
+
+ return new_lines
+
+
+def container_wrapper(
+ directive: SphinxDirective, literal_node: Node, caption: str,
+) -> nodes.container:
+ container_node = nodes.container('', literal_block=True,
+ classes=['literal-block-wrapper'])
+ parsed = directive.parse_text_to_nodes(caption, offset=directive.content_offset)
+ node = parsed[0]
+ if isinstance(node, nodes.system_message):
+ msg = __('Invalid caption: %s') % node.astext()
+ raise ValueError(msg)
+ if isinstance(node, nodes.Element):
+ caption_node = nodes.caption(node.rawsource, '', *node.children)
+ caption_node.source = literal_node.source
+ caption_node.line = literal_node.line
+ container_node += caption_node
+ container_node += literal_node
+ return container_node
+ raise RuntimeError # never reached
class CodeBlock(SphinxDirective):
@@ -36,33 +94,290 @@ class CodeBlock(SphinxDirective):
Directive for a code block with special highlighting or line numbering
settings.
"""
+
has_content = True
required_arguments = 0
optional_arguments = 1
final_argument_whitespace = False
- option_spec: ClassVar[OptionSpec] = {'force': directives.flag,
- 'linenos': directives.flag, 'dedent': optional_int, 'lineno-start':
- int, 'emphasize-lines': directives.unchanged_required, 'caption':
- directives.unchanged_required, 'class': directives.class_option,
- 'name': directives.unchanged}
+ option_spec: ClassVar[OptionSpec] = {
+ 'force': directives.flag,
+ 'linenos': directives.flag,
+ 'dedent': optional_int,
+ 'lineno-start': int,
+ 'emphasize-lines': directives.unchanged_required,
+ 'caption': directives.unchanged_required,
+ 'class': directives.class_option,
+ 'name': directives.unchanged,
+ }
+
+ def run(self) -> list[Node]:
+ document = self.state.document
+ code = '\n'.join(self.content)
+ location = self.state_machine.get_source_and_line(self.lineno)
+
+ linespec = self.options.get('emphasize-lines')
+ if linespec:
+ try:
+ nlines = len(self.content)
+ hl_lines = parselinenos(linespec, nlines)
+ if any(i >= nlines for i in hl_lines):
+ logger.warning(__('line number spec is out of range(1-%d): %r'),
+ nlines, self.options['emphasize-lines'],
+ location=location)
+
+ hl_lines = [x + 1 for x in hl_lines if x < nlines]
+ except ValueError as err:
+ return [document.reporter.warning(err, line=self.lineno)]
+ else:
+ hl_lines = None
+
+ if 'dedent' in self.options:
+ location = self.state_machine.get_source_and_line(self.lineno)
+ lines = code.splitlines(True)
+ lines = dedent_lines(lines, self.options['dedent'], location=location)
+ code = ''.join(lines)
+
+ literal: Element = nodes.literal_block(code, code)
+ if 'linenos' in self.options or 'lineno-start' in self.options:
+ literal['linenos'] = True
+ literal['classes'] += self.options.get('class', [])
+ literal['force'] = 'force' in self.options
+ if self.arguments:
+ # highlight language specified
+ literal['language'] = self.arguments[0]
+ else:
+ # no highlight language specified. Then this directive refers the current
+ # highlight setting via ``highlight`` directive or ``highlight_language``
+ # configuration.
+ literal['language'] = self.env.temp_data.get('highlight_language',
+ self.config.highlight_language)
+ extra_args = literal['highlight_args'] = {}
+ if hl_lines is not None:
+ extra_args['hl_lines'] = hl_lines
+ if 'lineno-start' in self.options:
+ extra_args['linenostart'] = self.options['lineno-start']
+ self.set_source_info(literal)
+
+ caption = self.options.get('caption')
+ if caption:
+ try:
+ literal = container_wrapper(self, literal, caption)
+ except ValueError as exc:
+ return [document.reporter.warning(exc, line=self.lineno)]
+
+ # literal will be note_implicit_target that is linked from caption and numref.
+ # when options['name'] is provided, it should be primary ID.
+ self.add_name(literal)
+
+ return [literal]
class LiteralIncludeReader:
- INVALID_OPTIONS_PAIR = [('lineno-match', 'lineno-start'), (
- 'lineno-match', 'append'), ('lineno-match', 'prepend'), (
- 'start-after', 'start-at'), ('end-before', 'end-at'), ('diff',
- 'pyobject'), ('diff', 'lineno-start'), ('diff', 'lineno-match'), (
- 'diff', 'lines'), ('diff', 'start-after'), ('diff', 'end-before'),
- ('diff', 'start-at'), ('diff', 'end-at')]
-
- def __init__(self, filename: str, options: dict[str, Any], config: Config
- ) ->None:
+ INVALID_OPTIONS_PAIR = [
+ ('lineno-match', 'lineno-start'),
+ ('lineno-match', 'append'),
+ ('lineno-match', 'prepend'),
+ ('start-after', 'start-at'),
+ ('end-before', 'end-at'),
+ ('diff', 'pyobject'),
+ ('diff', 'lineno-start'),
+ ('diff', 'lineno-match'),
+ ('diff', 'lines'),
+ ('diff', 'start-after'),
+ ('diff', 'end-before'),
+ ('diff', 'start-at'),
+ ('diff', 'end-at'),
+ ]
+
+ def __init__(self, filename: str, options: dict[str, Any], config: Config) -> None:
self.filename = filename
self.options = options
self.encoding = options.get('encoding', config.source_encoding)
self.lineno_start = self.options.get('lineno-start', 1)
+
self.parse_options()
+ def parse_options(self) -> None:
+ for option1, option2 in self.INVALID_OPTIONS_PAIR:
+ if option1 in self.options and option2 in self.options:
+ raise ValueError(__('Cannot use both "%s" and "%s" options') %
+ (option1, option2))
+
+ def read_file(
+ self, filename: str, location: tuple[str, int] | None = None,
+ ) -> list[str]:
+ try:
+ with open(filename, encoding=self.encoding, errors='strict') as f:
+ text = f.read()
+ if 'tab-width' in self.options:
+ text = text.expandtabs(self.options['tab-width'])
+
+ return text.splitlines(True)
+ except OSError as exc:
+ raise OSError(__('Include file %r not found or reading it failed') %
+ filename) from exc
+ except UnicodeError as exc:
+ raise UnicodeError(__('Encoding %r used for reading included file %r seems to '
+ 'be wrong, try giving an :encoding: option') %
+ (self.encoding, filename)) from exc
+
+ def read(self, location: tuple[str, int] | None = None) -> tuple[str, int]:
+ if 'diff' in self.options:
+ lines = self.show_diff()
+ else:
+ filters = [self.pyobject_filter,
+ self.start_filter,
+ self.end_filter,
+ self.lines_filter,
+ self.dedent_filter,
+ self.prepend_filter,
+ self.append_filter]
+ lines = self.read_file(self.filename, location=location)
+ for func in filters:
+ lines = func(lines, location=location)
+
+ return ''.join(lines), len(lines)
+
+ def show_diff(self, location: tuple[str, int] | None = None) -> list[str]:
+ new_lines = self.read_file(self.filename)
+ old_filename = self.options['diff']
+ old_lines = self.read_file(old_filename)
+ diff = unified_diff(old_lines, new_lines, str(old_filename), str(self.filename))
+ return list(diff)
+
+ def pyobject_filter(
+ self, lines: list[str], location: tuple[str, int] | None = None,
+ ) -> list[str]:
+ pyobject = self.options.get('pyobject')
+ if pyobject:
+ from sphinx.pycode import ModuleAnalyzer
+ analyzer = ModuleAnalyzer.for_file(self.filename, '')
+ tags = analyzer.find_tags()
+ if pyobject not in tags:
+ raise ValueError(__('Object named %r not found in include file %r') %
+ (pyobject, self.filename))
+ start = tags[pyobject][1]
+ end = tags[pyobject][2]
+ lines = lines[start - 1:end]
+ if 'lineno-match' in self.options:
+ self.lineno_start = start
+
+ return lines
+
+ def lines_filter(
+ self, lines: list[str], location: tuple[str, int] | None = None,
+ ) -> list[str]:
+ linespec = self.options.get('lines')
+ if linespec:
+ linelist = parselinenos(linespec, len(lines))
+ if any(i >= len(lines) for i in linelist):
+ logger.warning(__('line number spec is out of range(1-%d): %r'),
+ len(lines), linespec, location=location)
+
+ if 'lineno-match' in self.options:
+ # make sure the line list is not "disjoint".
+ first = linelist[0]
+ if all(first + i == n for i, n in enumerate(linelist)):
+ self.lineno_start += linelist[0]
+ else:
+ raise ValueError(__('Cannot use "lineno-match" with a disjoint '
+ 'set of "lines"'))
+
+ lines = [lines[n] for n in linelist if n < len(lines)]
+ if not lines:
+ raise ValueError(__('Line spec %r: no lines pulled from include file %r') %
+ (linespec, self.filename))
+
+ return lines
+
+ def start_filter(
+ self, lines: list[str], location: tuple[str, int] | None = None,
+ ) -> list[str]:
+ if 'start-at' in self.options:
+ start = self.options.get('start-at')
+ inclusive = False
+ elif 'start-after' in self.options:
+ start = self.options.get('start-after')
+ inclusive = True
+ else:
+ start = None
+
+ if start:
+ for lineno, line in enumerate(lines):
+ if start in line:
+ if inclusive:
+ if 'lineno-match' in self.options:
+ self.lineno_start += lineno + 1
+
+ return lines[lineno + 1:]
+ else:
+ if 'lineno-match' in self.options:
+ self.lineno_start += lineno
+
+ return lines[lineno:]
+
+ if inclusive is True:
+ raise ValueError('start-after pattern not found: %s' % start)
+ else:
+ raise ValueError('start-at pattern not found: %s' % start)
+
+ return lines
+
+ def end_filter(
+ self, lines: list[str], location: tuple[str, int] | None = None,
+ ) -> list[str]:
+ if 'end-at' in self.options:
+ end = self.options.get('end-at')
+ inclusive = True
+ elif 'end-before' in self.options:
+ end = self.options.get('end-before')
+ inclusive = False
+ else:
+ end = None
+
+ if end:
+ for lineno, line in enumerate(lines):
+ if end in line:
+ if inclusive:
+ return lines[:lineno + 1]
+ else:
+ if lineno == 0:
+ pass # end-before ignores first line
+ else:
+ return lines[:lineno]
+ if inclusive is True:
+ raise ValueError('end-at pattern not found: %s' % end)
+ else:
+ raise ValueError('end-before pattern not found: %s' % end)
+
+ return lines
+
+ def prepend_filter(
+ self, lines: list[str], location: tuple[str, int] | None = None,
+ ) -> list[str]:
+ prepend = self.options.get('prepend')
+ if prepend:
+ lines.insert(0, prepend + '\n')
+
+ return lines
+
+ def append_filter(
+ self, lines: list[str], location: tuple[str, int] | None = None,
+ ) -> list[str]:
+ append = self.options.get('append')
+ if append:
+ lines.append(append + '\n')
+
+ return lines
+
+ def dedent_filter(
+ self, lines: list[str], location: tuple[str, int] | None = None,
+ ) -> list[str]:
+ if 'dedent' in self.options:
+ return dedent_lines(lines, self.options.get('dedent'), location=location)
+ else:
+ return lines
+
class LiteralInclude(SphinxDirective):
"""
@@ -70,20 +385,95 @@ class LiteralInclude(SphinxDirective):
not found, and does not raise errors. Also has several options for
selecting what to include.
"""
+
has_content = False
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
- option_spec: ClassVar[OptionSpec] = {'dedent': optional_int, 'linenos':
- directives.flag, 'lineno-start': int, 'lineno-match': directives.
- flag, 'tab-width': int, 'language': directives.unchanged_required,
- 'force': directives.flag, 'encoding': directives.encoding,
- 'pyobject': directives.unchanged_required, 'lines': directives.
- unchanged_required, 'start-after': directives.unchanged_required,
- 'end-before': directives.unchanged_required, 'start-at': directives
- .unchanged_required, 'end-at': directives.unchanged_required,
- 'prepend': directives.unchanged_required, 'append': directives.
- unchanged_required, 'emphasize-lines': directives.
- unchanged_required, 'caption': directives.unchanged, 'class':
- directives.class_option, 'name': directives.unchanged, 'diff':
- directives.unchanged_required}
+ option_spec: ClassVar[OptionSpec] = {
+ 'dedent': optional_int,
+ 'linenos': directives.flag,
+ 'lineno-start': int,
+ 'lineno-match': directives.flag,
+ 'tab-width': int,
+ 'language': directives.unchanged_required,
+ 'force': directives.flag,
+ 'encoding': directives.encoding,
+ 'pyobject': directives.unchanged_required,
+ 'lines': directives.unchanged_required,
+ 'start-after': directives.unchanged_required,
+ 'end-before': directives.unchanged_required,
+ 'start-at': directives.unchanged_required,
+ 'end-at': directives.unchanged_required,
+ 'prepend': directives.unchanged_required,
+ 'append': directives.unchanged_required,
+ 'emphasize-lines': directives.unchanged_required,
+ 'caption': directives.unchanged,
+ 'class': directives.class_option,
+ 'name': directives.unchanged,
+ 'diff': directives.unchanged_required,
+ }
+
+ def run(self) -> list[Node]:
+ document = self.state.document
+ if not document.settings.file_insertion_enabled:
+ return [document.reporter.warning('File insertion disabled',
+ line=self.lineno)]
+ # convert options['diff'] to absolute path
+ if 'diff' in self.options:
+ _, path = self.env.relfn2path(self.options['diff'])
+ self.options['diff'] = path
+
+ try:
+ location = self.state_machine.get_source_and_line(self.lineno)
+ rel_filename, filename = self.env.relfn2path(self.arguments[0])
+ self.env.note_dependency(rel_filename)
+
+ reader = LiteralIncludeReader(filename, self.options, self.config)
+ text, lines = reader.read(location=location)
+
+ retnode: Element = nodes.literal_block(text, text, source=filename)
+ retnode['force'] = 'force' in self.options
+ self.set_source_info(retnode)
+ if self.options.get('diff'): # if diff is set, set udiff
+ retnode['language'] = 'udiff'
+ elif 'language' in self.options:
+ retnode['language'] = self.options['language']
+ if ('linenos' in self.options or 'lineno-start' in self.options or
+ 'lineno-match' in self.options):
+ retnode['linenos'] = True
+ retnode['classes'] += self.options.get('class', [])
+ extra_args = retnode['highlight_args'] = {}
+ if 'emphasize-lines' in self.options:
+ hl_lines = parselinenos(self.options['emphasize-lines'], lines)
+ if any(i >= lines for i in hl_lines):
+ logger.warning(__('line number spec is out of range(1-%d): %r'),
+ lines, self.options['emphasize-lines'],
+ location=location)
+ extra_args['hl_lines'] = [x + 1 for x in hl_lines if x < lines]
+ extra_args['linenostart'] = reader.lineno_start
+
+ if 'caption' in self.options:
+ caption = self.options['caption'] or self.arguments[0]
+ retnode = container_wrapper(self, retnode, caption)
+
+ # retnode will be note_implicit_target that is linked from caption and numref.
+ # when options['name'] is provided, it should be primary ID.
+ self.add_name(retnode)
+
+ return [retnode]
+ except Exception as exc:
+ return [document.reporter.warning(exc, line=self.lineno)]
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ directives.register_directive('highlight', Highlight)
+ directives.register_directive('code-block', CodeBlock)
+ directives.register_directive('sourcecode', CodeBlock)
+ directives.register_directive('literalinclude', LiteralInclude)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/directives/other.py b/sphinx/directives/other.py
index 1a6198deb..18fdff194 100644
--- a/sphinx/directives/other.py
+++ b/sphinx/directives/other.py
@@ -1,14 +1,17 @@
from __future__ import annotations
+
import re
from os.path import abspath, relpath
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar, cast
+
from docutils import nodes
from docutils.parsers.rst import directives
from docutils.parsers.rst.directives.admonitions import BaseAdmonition
from docutils.parsers.rst.directives.misc import Class
from docutils.parsers.rst.directives.misc import Include as BaseInclude
from docutils.statemachine import StateMachine
+
from sphinx import addnodes
from sphinx.domains.std import StandardDomain
from sphinx.locale import _, __
@@ -16,35 +19,160 @@ from sphinx.util import docname_join, logging, url_re
from sphinx.util.docutils import SphinxDirective
from sphinx.util.matching import Matcher, patfilter
from sphinx.util.nodes import explicit_title_re
+
if TYPE_CHECKING:
from collections.abc import Sequence
+
from docutils.nodes import Element, Node
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata, OptionSpec
-glob_re = re.compile('.*[*?\\[].*')
+
+
+glob_re = re.compile(r'.*[*?\[].*')
logger = logging.getLogger(__name__)
+def int_or_nothing(argument: str) -> int:
+ if not argument:
+ return 999
+ return int(argument)
+
+
class TocTree(SphinxDirective):
"""
Directive to notify Sphinx about the hierarchical structure of the docs,
and to include a table-of-contents like tree in the current document.
"""
+
has_content = True
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = False
- option_spec = {'maxdepth': int, 'name': directives.unchanged, 'class':
- directives.class_option, 'caption': directives.unchanged_required,
- 'glob': directives.flag, 'hidden': directives.flag, 'includehidden':
- directives.flag, 'numbered': int_or_nothing, 'titlesonly':
- directives.flag, 'reversed': directives.flag}
+ option_spec = {
+ 'maxdepth': int,
+ 'name': directives.unchanged,
+ 'class': directives.class_option,
+ 'caption': directives.unchanged_required,
+ 'glob': directives.flag,
+ 'hidden': directives.flag,
+ 'includehidden': directives.flag,
+ 'numbered': int_or_nothing,
+ 'titlesonly': directives.flag,
+ 'reversed': directives.flag,
+ }
+
+ def run(self) -> list[Node]:
+ subnode = addnodes.toctree()
+ subnode['parent'] = self.env.docname
+
+ # (title, ref) pairs, where ref may be a document, or an external link,
+ # and title may be None if the document's title is to be used
+ subnode['entries'] = []
+ subnode['includefiles'] = []
+ subnode['maxdepth'] = self.options.get('maxdepth', -1)
+ subnode['caption'] = self.options.get('caption')
+ subnode['glob'] = 'glob' in self.options
+ subnode['hidden'] = 'hidden' in self.options
+ subnode['includehidden'] = 'includehidden' in self.options
+ subnode['numbered'] = self.options.get('numbered', 0)
+ subnode['titlesonly'] = 'titlesonly' in self.options
+ self.set_source_info(subnode)
+ self.parse_content(subnode)
+
+ wrappernode = nodes.compound(
+ classes=['toctree-wrapper', *self.options.get('class', ())],
+ )
+ wrappernode.append(subnode)
+ self.add_name(wrappernode)
+ return [wrappernode]
- def parse_content(self, toctree: addnodes.toctree) ->None:
+ def parse_content(self, toctree: addnodes.toctree) -> None:
"""
Populate ``toctree['entries']`` and ``toctree['includefiles']`` from content.
"""
- pass
+ generated_docnames = frozenset(StandardDomain._virtual_doc_names)
+ suffixes = self.config.source_suffix
+ current_docname = self.env.docname
+ glob = toctree['glob']
+
+ # glob target documents
+ all_docnames = self.env.found_docs.copy() | generated_docnames
+ all_docnames.remove(current_docname) # remove current document
+ frozen_all_docnames = frozenset(all_docnames)
+
+ excluded = Matcher(self.config.exclude_patterns)
+ for entry in self.content:
+ if not entry:
+ continue
+
+ # look for explicit titles ("Some Title <document>")
+ explicit = explicit_title_re.match(entry)
+ url_match = url_re.match(entry) is not None
+ if glob and glob_re.match(entry) and not explicit and not url_match:
+ pat_name = docname_join(current_docname, entry)
+ doc_names = sorted(
+ docname for docname in patfilter(all_docnames, pat_name)
+ # don't include generated documents in globs
+ if docname not in generated_docnames
+ )
+ if not doc_names:
+ logger.warning(
+ __("toctree glob pattern %r didn't match any documents"),
+ entry, location=toctree)
+
+ for docname in doc_names:
+ all_docnames.remove(docname) # don't include it again
+ toctree['entries'].append((None, docname))
+ toctree['includefiles'].append(docname)
+ continue
+
+ if explicit:
+ ref = explicit.group(2)
+ title = explicit.group(1)
+ docname = ref
+ else:
+ ref = docname = entry
+ title = None
+
+ # remove suffixes (backwards compatibility)
+ for suffix in suffixes:
+ if docname.endswith(suffix):
+ docname = docname.removesuffix(suffix)
+ break
+
+ # absolutise filenames
+ docname = docname_join(current_docname, docname)
+ if url_match or ref == 'self':
+ toctree['entries'].append((title, ref))
+ continue
+
+ if docname not in frozen_all_docnames:
+ if excluded(str(self.env.doc2path(docname, False))):
+ message = __('toctree contains reference to excluded document %r')
+ subtype = 'excluded'
+ else:
+ message = __('toctree contains reference to nonexisting document %r')
+ subtype = 'not_readable'
+
+ logger.warning(message, docname, type='toc', subtype=subtype,
+ location=toctree)
+ self.env.note_reread()
+ continue
+
+ if docname in all_docnames:
+ all_docnames.remove(docname)
+ else:
+ logger.warning(__('duplicated entry found in toctree: %s'), docname,
+ location=toctree)
+
+ toctree['entries'].append((title, docname))
+ toctree['includefiles'].append(docname)
+
+ # entries contains all entries (self references, external links etc.)
+ if 'reversed' in self.options:
+ toctree['entries'] = list(reversed(toctree['entries']))
+ toctree['includefiles'] = list(reversed(toctree['includefiles']))
class Author(SphinxDirective):
@@ -52,17 +180,41 @@ class Author(SphinxDirective):
Directive to give the name of the author of the current document
or section. Shown in the output only if the show_authors option is on.
"""
+
has_content = False
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
option_spec: ClassVar[OptionSpec] = {}
+ def run(self) -> list[Node]:
+ if not self.config.show_authors:
+ return []
+ para: Element = nodes.paragraph(translatable=False)
+ emph = nodes.emphasis()
+ para += emph
+ if self.name == 'sectionauthor':
+ text = _('Section author: ')
+ elif self.name == 'moduleauthor':
+ text = _('Module author: ')
+ elif self.name == 'codeauthor':
+ text = _('Code author: ')
+ else:
+ text = _('Author: ')
+ emph += nodes.Text(text)
+ inodes, messages = self.parse_inline(self.arguments[0])
+ emph.extend(inodes)
-class SeeAlso(BaseAdmonition):
+ ret: list[Node] = [para]
+ ret += messages
+ return ret
+
+
+class SeeAlso(BaseAdmonition): # type: ignore[misc]
"""
An admonition mentioning things to look at as reference.
"""
+
node_class = addnodes.seealso
@@ -70,59 +222,230 @@ class TabularColumns(SphinxDirective):
"""
Directive to give an explicit tabulary column definition to LaTeX.
"""
+
has_content = False
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
option_spec: ClassVar[OptionSpec] = {}
+ def run(self) -> list[Node]:
+ node = addnodes.tabular_col_spec()
+ node['spec'] = self.arguments[0]
+ self.set_source_info(node)
+ return [node]
+
class Centered(SphinxDirective):
"""
Directive to create a centered line of bold text.
"""
+
has_content = False
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
option_spec: ClassVar[OptionSpec] = {}
+ def run(self) -> list[Node]:
+ if not self.arguments:
+ return []
+ subnode: Element = addnodes.centered()
+ inodes, messages = self.parse_inline(self.arguments[0])
+ subnode.extend(inodes)
+
+ ret: list[Node] = [subnode]
+ ret += messages
+ return ret
+
class Acks(SphinxDirective):
"""
Directive for a list of names.
"""
+
has_content = True
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = False
option_spec: ClassVar[OptionSpec] = {}
+ def run(self) -> list[Node]:
+ children = self.parse_content_to_nodes()
+ if len(children) != 1 or not isinstance(children[0], nodes.bullet_list):
+ logger.warning(__('.. acks content is not a list'),
+ location=(self.env.docname, self.lineno))
+ return []
+ return [addnodes.acks('', *children)]
+
class HList(SphinxDirective):
"""
Directive for a list that gets compacted horizontally.
"""
+
has_content = True
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = False
- option_spec: ClassVar[OptionSpec] = {'columns': int}
+ option_spec: ClassVar[OptionSpec] = {
+ 'columns': int,
+ }
+
+ def run(self) -> list[Node]:
+ ncolumns = self.options.get('columns', 2)
+ children = self.parse_content_to_nodes()
+ if len(children) != 1 or not isinstance(children[0], nodes.bullet_list):
+ logger.warning(__('.. hlist content is not a list'),
+ location=(self.env.docname, self.lineno))
+ return []
+ fulllist = children[0]
+ # create a hlist node where the items are distributed
+ npercol, nmore = divmod(len(fulllist), ncolumns)
+ index = 0
+ newnode = addnodes.hlist()
+ newnode['ncolumns'] = str(ncolumns)
+ for column in range(ncolumns):
+ endindex = index + ((npercol + 1) if column < nmore else npercol)
+ bullet_list = nodes.bullet_list()
+ bullet_list += fulllist.children[index:endindex]
+ newnode += addnodes.hlistcol('', bullet_list)
+ index = endindex
+ return [newnode]
class Only(SphinxDirective):
"""
Directive to only include text if the given tag(s) are enabled.
"""
+
has_content = True
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
option_spec: ClassVar[OptionSpec] = {}
+ def run(self) -> list[Node]:
+ node = addnodes.only()
+ node.document = self.state.document
+ self.set_source_info(node)
+ node['expr'] = self.arguments[0]
+
+ # Same as util.nested_parse_with_titles but try to handle nested
+ # sections which should be raised higher up the doctree.
+ memo: Any = self.state.memo
+ surrounding_title_styles = memo.title_styles
+ surrounding_section_level = memo.section_level
+ memo.title_styles = []
+ memo.section_level = 0
+ try:
+ self.state.nested_parse(self.content, self.content_offset,
+ node, match_titles=True)
+ title_styles = memo.title_styles
+ if (not surrounding_title_styles or
+ not title_styles or
+ title_styles[0] not in surrounding_title_styles or
+ not self.state.parent):
+ # No nested sections so no special handling needed.
+ return [node]
+ # Calculate the depths of the current and nested sections.
+ current_depth = 0
+ parent = self.state.parent
+ while parent:
+ current_depth += 1
+ parent = parent.parent
+ current_depth -= 2
+ title_style = title_styles[0]
+ nested_depth = len(surrounding_title_styles)
+ if title_style in surrounding_title_styles:
+ nested_depth = surrounding_title_styles.index(title_style)
+ # Use these depths to determine where the nested sections should
+ # be placed in the doctree.
+ n_sects_to_raise = current_depth - nested_depth + 1
+ parent = cast(nodes.Element, self.state.parent)
+ for _i in range(n_sects_to_raise):
+ if parent.parent:
+ parent = parent.parent
+ parent.append(node)
+ return []
+ finally:
+ memo.title_styles = surrounding_title_styles
+ memo.section_level = surrounding_section_level
+
class Include(BaseInclude, SphinxDirective):
"""
Like the standard "Include" directive, but interprets absolute paths
"correctly", i.e. relative to source directory.
"""
+
+ def run(self) -> Sequence[Node]:
+
+ # To properly emit "include-read" events from included RST text,
+ # we must patch the ``StateMachine.insert_input()`` method.
+ # In the future, docutils will hopefully offer a way for Sphinx
+ # to provide the RST parser to use
+ # when parsing RST text that comes in via Include directive.
+ def _insert_input(include_lines: list[str], source: str) -> None:
+ # First, we need to combine the lines back into text so that
+ # we can send it with the include-read event.
+ # In docutils 0.18 and later, there are two lines at the end
+ # that act as markers.
+ # We must preserve them and leave them out of the include-read event:
+ text = "\n".join(include_lines[:-2])
+
+ path = Path(relpath(abspath(source), start=self.env.srcdir))
+ docname = self.env.docname
+
+ # Emit the "include-read" event
+ arg = [text]
+ self.env.app.events.emit('include-read', path, docname, arg)
+ text = arg[0]
+
+ # Split back into lines and reattach the two marker lines
+ include_lines = text.splitlines() + include_lines[-2:]
+
+ # Call the parent implementation.
+ # Note that this snake does not eat its tail because we patch
+ # the *Instance* method and this call is to the *Class* method.
+ return StateMachine.insert_input(self.state_machine, include_lines, source)
+
+ # Only enable this patch if there are listeners for 'include-read'.
+ if self.env.app.events.listeners.get('include-read'):
+ # See https://github.com/python/mypy/issues/2427 for details on the mypy issue
+ self.state_machine.insert_input = _insert_input
+
+ if self.arguments[0].startswith('<') and \
+ self.arguments[0].endswith('>'):
+ # docutils "standard" includes, do not do path processing
+ return super().run()
+ rel_filename, filename = self.env.relfn2path(self.arguments[0])
+ self.arguments[0] = filename
+ self.env.note_included(filename)
+ return super().run()
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ directives.register_directive('toctree', TocTree)
+ directives.register_directive('sectionauthor', Author)
+ directives.register_directive('moduleauthor', Author)
+ directives.register_directive('codeauthor', Author)
+ directives.register_directive('seealso', SeeAlso)
+ directives.register_directive('tabularcolumns', TabularColumns)
+ directives.register_directive('centered', Centered)
+ directives.register_directive('acks', Acks)
+ directives.register_directive('hlist', HList)
+ directives.register_directive('only', Only)
+ directives.register_directive('include', Include)
+
+ # register the standard rst class directive under a different name
+ # only for backwards compatibility now
+ directives.register_directive('cssclass', Class)
+ # new standard name when default-domain with "class" is in effect
+ directives.register_directive('rst-class', Class)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/directives/patches.py b/sphinx/directives/patches.py
index 8dde54aa7..55716ba89 100644
--- a/sphinx/directives/patches.py
+++ b/sphinx/directives/patches.py
@@ -1,13 +1,16 @@
from __future__ import annotations
+
import os
from os import path
from typing import TYPE_CHECKING, ClassVar, cast
+
from docutils import nodes
from docutils.nodes import Node, make_id
from docutils.parsers.rst import directives
from docutils.parsers.rst.directives import images, tables
from docutils.parsers.rst.directives.misc import Meta
from docutils.parsers.rst.roles import set_classes
+
from sphinx.directives import optional_int
from sphinx.domains.math import MathDomain
from sphinx.locale import __
@@ -15,53 +18,199 @@ from sphinx.util import logging
from sphinx.util.docutils import SphinxDirective
from sphinx.util.nodes import set_source_info
from sphinx.util.osutil import SEP, os_path, relpath
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata, OptionSpec
+
+
logger = logging.getLogger(__name__)
-class Figure(images.Figure):
+class Figure(images.Figure): # type: ignore[misc]
"""The figure directive which applies `:name:` option to the figure node
instead of the image node.
"""
+ def run(self) -> list[Node]:
+ name = self.options.pop('name', None)
+ result = super().run()
+ if len(result) == 2 or isinstance(result[0], nodes.system_message):
+ return result
+
+ assert len(result) == 1
+ figure_node = cast(nodes.figure, result[0])
+ if name:
+ # set ``name`` to figure_node if given
+ self.options['name'] = name
+ self.add_name(figure_node)
+
+ # copy lineno from image node
+ if figure_node.line is None and len(figure_node) == 2:
+ caption = cast(nodes.caption, figure_node[1])
+ figure_node.line = caption.line
-class CSVTable(tables.CSVTable):
+ return [figure_node]
+
+
+class CSVTable(tables.CSVTable): # type: ignore[misc]
"""The csv-table directive which searches a CSV file from Sphinx project's source
directory when an absolute path is given via :file: option.
"""
+ def run(self) -> list[Node]:
+ if 'file' in self.options and self.options['file'].startswith((SEP, os.sep)):
+ env = self.state.document.settings.env
+ filename = self.options['file']
+ if path.exists(filename):
+ logger.warning(__('":file:" option for csv-table directive now recognizes '
+ 'an absolute path as a relative path from source directory. '
+ 'Please update your document.'),
+ location=(env.docname, self.lineno))
+ else:
+ abspath = path.join(env.srcdir, os_path(self.options['file'][1:]))
+ docdir = path.dirname(env.doc2path(env.docname))
+ self.options['file'] = relpath(abspath, docdir)
+
+ return super().run()
+
class Code(SphinxDirective):
"""Parse and mark up content of a code block.
This is compatible with docutils' :rst:dir:`code` directive.
"""
+
optional_arguments = 1
- option_spec: ClassVar[OptionSpec] = {'class': directives.class_option,
- 'force': directives.flag, 'name': directives.unchanged,
- 'number-lines': optional_int}
+ option_spec: ClassVar[OptionSpec] = {
+ 'class': directives.class_option,
+ 'force': directives.flag,
+ 'name': directives.unchanged,
+ 'number-lines': optional_int,
+ }
has_content = True
+ def run(self) -> list[Node]:
+ self.assert_has_content()
+
+ set_classes(self.options)
+ code = '\n'.join(self.content)
+ node = nodes.literal_block(code, code,
+ classes=self.options.get('classes', []),
+ force='force' in self.options,
+ highlight_args={})
+ self.add_name(node)
+ set_source_info(self, node)
+
+ if self.arguments:
+ # highlight language specified
+ node['language'] = self.arguments[0]
+ else:
+ # no highlight language specified. Then this directive refers the current
+ # highlight setting via ``highlight`` directive or ``highlight_language``
+ # configuration.
+ node['language'] = self.env.temp_data.get('highlight_language',
+ self.config.highlight_language)
+
+ if 'number-lines' in self.options:
+ node['linenos'] = True
+
+ # if number given, treat as lineno-start.
+ if self.options['number-lines']:
+ node['highlight_args']['linenostart'] = self.options['number-lines']
+
+ return [node]
+
class MathDirective(SphinxDirective):
has_content = True
required_arguments = 0
optional_arguments = 1
final_argument_whitespace = True
- option_spec: ClassVar[OptionSpec] = {'label': directives.unchanged,
- 'name': directives.unchanged, 'class': directives.class_option,
- 'nowrap': directives.flag}
+ option_spec: ClassVar[OptionSpec] = {
+ 'label': directives.unchanged,
+ 'name': directives.unchanged,
+ 'class': directives.class_option,
+ 'nowrap': directives.flag,
+ }
+
+ def run(self) -> list[Node]:
+ latex = '\n'.join(self.content)
+ if self.arguments and self.arguments[0]:
+ latex = self.arguments[0] + '\n\n' + latex
+ label = self.options.get('label', self.options.get('name'))
+ node = nodes.math_block(latex, latex,
+ classes=self.options.get('class', []),
+ docname=self.env.docname,
+ number=None,
+ label=label,
+ nowrap='nowrap' in self.options)
+ self.add_name(node)
+ self.set_source_info(node)
+
+ ret: list[Node] = [node]
+ self.add_target(ret)
+ return ret
+
+ def add_target(self, ret: list[Node]) -> None:
+ node = cast(nodes.math_block, ret[0])
+
+ # assign label automatically if math_number_all enabled
+ if node['label'] == '' or (self.config.math_number_all and not node['label']):
+ seq = self.env.new_serialno('sphinx.ext.math#equations')
+ node['label'] = "%s:%d" % (self.env.docname, seq)
+
+ # no targets and numbers are needed
+ if not node['label']:
+ return
+
+ # register label to domain
+ domain = cast(MathDomain, self.env.get_domain('math'))
+ domain.note_equation(self.env.docname, node['label'], location=node)
+ node['number'] = domain.get_equation_number_for(node['label'])
+
+ # add target node
+ node_id = make_id('equation-%s' % node['label'])
+ target = nodes.target('', '', ids=[node_id])
+ self.state.document.note_explicit_target(target)
+ ret.insert(0, target)
class Rubric(SphinxDirective):
"""A patch of the docutils' :rst:dir:`rubric` directive,
which adds a level option to specify the heading level of the rubric.
"""
+
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
- option_spec = {'class': directives.class_option, 'name': directives.
- unchanged, 'heading-level': lambda c: directives.choice(c, ('1',
- '2', '3', '4', '5', '6'))}
+ option_spec = {
+ 'class': directives.class_option,
+ 'name': directives.unchanged,
+ 'heading-level': lambda c: directives.choice(c, ('1', '2', '3', '4', '5', '6')),
+ }
+
+ def run(self) -> list[nodes.rubric | nodes.system_message]:
+ set_classes(self.options)
+ rubric_text = self.arguments[0]
+ textnodes, messages = self.parse_inline(rubric_text, lineno=self.lineno)
+ if 'heading-level' in self.options:
+ self.options['heading-level'] = int(self.options['heading-level'])
+ rubric = nodes.rubric(rubric_text, '', *textnodes, **self.options)
+ self.add_name(rubric)
+ return [rubric, *messages]
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ directives.register_directive('figure', Figure)
+ directives.register_directive('meta', Meta)
+ directives.register_directive('csv-table', CSVTable)
+ directives.register_directive('code', Code)
+ directives.register_directive('math', MathDirective)
+ directives.register_directive('rubric', Rubric)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/domains/_index.py b/sphinx/domains/_index.py
index ef50b53cd..1598d2f20 100644
--- a/sphinx/domains/_index.py
+++ b/sphinx/domains/_index.py
@@ -1,10 +1,15 @@
"""Domain indices."""
+
from __future__ import annotations
+
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, NamedTuple
+
from sphinx.errors import SphinxError
+
if TYPE_CHECKING:
from collections.abc import Iterable
+
from sphinx.domains import Domain
@@ -40,21 +45,22 @@ class Index(ABC):
Index pages can be referred by domain name and index name via
:rst:role:`ref` role.
"""
+
name: str
localname: str
shortname: str | None = None
- def __init__(self, domain: Domain) ->None:
+ def __init__(self, domain: Domain) -> None:
if not self.name or self.localname is None:
- msg = (
- f'Index subclass {self.__class__.__name__} has no valid name or localname'
- )
+ msg = f'Index subclass {self.__class__.__name__} has no valid name or localname'
raise SphinxError(msg)
self.domain = domain
@abstractmethod
- def generate(self, docnames: (Iterable[str] | None)=None) ->tuple[list[
- tuple[str, list[IndexEntry]]], bool]:
+ def generate(
+ self,
+ docnames: Iterable[str] | None = None,
+ ) -> tuple[list[tuple[str, list[IndexEntry]]], bool]:
"""Get entries for the index.
If ``docnames`` is given, restrict to entries referring to these
@@ -104,4 +110,4 @@ class Index(ABC):
Qualifier and description are not rendered for some output formats such
as LaTeX.
"""
- pass
+ raise NotImplementedError
diff --git a/sphinx/domains/c/_ast.py b/sphinx/domains/c/_ast.py
index 91f418235..5147a4598 100644
--- a/sphinx/domains/c/_ast.py
+++ b/sphinx/domains/c/_ast.py
@@ -1,98 +1,255 @@
from __future__ import annotations
+
import sys
import warnings
from typing import TYPE_CHECKING, Any, Union, cast
+
from docutils import nodes
+
from sphinx import addnodes
from sphinx.domains.c._ids import _id_prefix, _max_id
-from sphinx.util.cfamily import ASTAttributeList, ASTBaseBase, ASTBaseParenExprList, UnsupportedMultiCharacterCharLiteral, verify_description_mode
+from sphinx.util.cfamily import (
+ ASTAttributeList,
+ ASTBaseBase,
+ ASTBaseParenExprList,
+ UnsupportedMultiCharacterCharLiteral,
+ verify_description_mode,
+)
+
if TYPE_CHECKING:
from typing import TypeAlias
+
from docutils.nodes import Element, Node, TextElement
+
from sphinx.domains.c._symbol import Symbol
from sphinx.environment import BuildEnvironment
from sphinx.util.cfamily import StringifyTransform
-DeclarationType: TypeAlias = Union['ASTStruct', 'ASTUnion', 'ASTEnum',
- 'ASTEnumerator', 'ASTType', 'ASTTypeWithInit', 'ASTMacro']
+
+DeclarationType: TypeAlias = Union[ # NoQA: UP007
+ "ASTStruct", "ASTUnion", "ASTEnum", "ASTEnumerator",
+ "ASTType", "ASTTypeWithInit", "ASTMacro",
+]
class ASTBase(ASTBaseBase):
- pass
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ raise NotImplementedError(repr(self))
-class ASTIdentifier(ASTBaseBase):
+# Names
+################################################################################
- def __init__(self, name: str) ->None:
+class ASTIdentifier(ASTBaseBase):
+ def __init__(self, name: str) -> None:
if not isinstance(name, str) or len(name) == 0:
raise AssertionError
self.name = sys.intern(name)
self.is_anonymous = name[0] == '@'
- def __eq__(self, other: object) ->bool:
+ # ASTBaseBase already implements this method,
+ # but specialising it here improves performance
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTIdentifier):
return NotImplemented
return self.name == other.name
- def __str__(self) ->str:
+ def is_anon(self) -> bool:
+ return self.is_anonymous
+
+ # and this is where we finally make a difference between __str__ and the display string
+
+ def __str__(self) -> str:
return self.name
+ def get_display_string(self) -> str:
+ return "[anonymous]" if self.is_anonymous else self.name
+
+ def describe_signature(self, signode: TextElement, mode: str, env: BuildEnvironment,
+ prefix: str, symbol: Symbol) -> None:
+ # note: slightly different signature of describe_signature due to the prefix
+ verify_description_mode(mode)
+ if self.is_anonymous:
+ node = addnodes.desc_sig_name(text="[anonymous]")
+ else:
+ node = addnodes.desc_sig_name(self.name, self.name)
+ if mode == 'markType':
+ targetText = prefix + self.name
+ pnode = addnodes.pending_xref('', refdomain='c',
+ reftype='identifier',
+ reftarget=targetText, modname=None,
+ classname=None)
+ pnode['c:parent_key'] = symbol.get_lookup_key()
+ pnode += node
+ signode += pnode
+ elif mode == 'lastIsName':
+ nameNode = addnodes.desc_name()
+ nameNode += node
+ signode += nameNode
+ elif mode == 'noneIsName':
+ signode += node
+ else:
+ raise Exception('Unknown description mode: %s' % mode)
+
+ @property
+ def identifier(self) -> str:
+ warnings.warn(
+ '`ASTIdentifier.identifier` is deprecated, use `ASTIdentifier.name` instead',
+ DeprecationWarning, stacklevel=2,
+ )
+ return self.name
-class ASTNestedName(ASTBase):
- def __init__(self, names: list[ASTIdentifier], rooted: bool) ->None:
+class ASTNestedName(ASTBase):
+ def __init__(self, names: list[ASTIdentifier], rooted: bool) -> None:
assert len(names) > 0
self.names = names
self.rooted = rooted
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNestedName):
return NotImplemented
return self.names == other.names and self.rooted == other.rooted
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.names, self.rooted))
+ @property
+ def name(self) -> ASTNestedName:
+ return self
+
+ def get_id(self, version: int) -> str:
+ return '.'.join(str(n) for n in self.names)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = '.'.join(transform(n) for n in self.names)
+ if self.rooted:
+ return '.' + res
+ else:
+ return res
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ # just print the name part, with template args, not template params
+ if mode == 'noneIsName':
+ if self.rooted:
+ unreachable = "Can this happen?"
+ raise AssertionError(unreachable) # TODO
+ signode += nodes.Text('.')
+ for i in range(len(self.names)):
+ if i != 0:
+ unreachable = "Can this happen?"
+ raise AssertionError(unreachable) # TODO
+ signode += nodes.Text('.')
+ n = self.names[i]
+ n.describe_signature(signode, mode, env, '', symbol)
+ elif mode == 'param':
+ assert not self.rooted, str(self)
+ assert len(self.names) == 1
+ self.names[0].describe_signature(signode, 'noneIsName', env, '', symbol)
+ elif mode in ('markType', 'lastIsName', 'markName'):
+ # Each element should be a pending xref targeting the complete
+ # prefix.
+ prefix = ''
+ first = True
+ names = self.names[:-1] if mode == 'lastIsName' else self.names
+ # If lastIsName, then wrap all of the prefix in a desc_addname,
+ # else append directly to signode.
+ # TODO: also for C?
+ # NOTE: Breathe previously relied on the prefix being in the desc_addname node,
+ # so it can remove it in inner declarations.
+ dest = signode
+ if mode == 'lastIsName':
+ dest = addnodes.desc_addname()
+ if self.rooted:
+ prefix += '.'
+ if mode == 'lastIsName' and len(names) == 0:
+ signode += addnodes.desc_sig_punctuation('.', '.')
+ else:
+ dest += addnodes.desc_sig_punctuation('.', '.')
+ for i in range(len(names)):
+ ident = names[i]
+ if not first:
+ dest += addnodes.desc_sig_punctuation('.', '.')
+ prefix += '.'
+ first = False
+ txt_ident = str(ident)
+ if txt_ident != '':
+ ident.describe_signature(dest, 'markType', env, prefix, symbol)
+ prefix += txt_ident
+ if mode == 'lastIsName':
+ if len(self.names) > 1:
+ dest += addnodes.desc_sig_punctuation('.', '.')
+ signode += dest
+ self.names[-1].describe_signature(signode, mode, env, '', symbol)
+ else:
+ raise Exception('Unknown description mode: %s' % mode)
+
+
+################################################################################
+# Expressions
+################################################################################
class ASTExpression(ASTBase):
pass
+# Primary expressions
+################################################################################
+
class ASTLiteral(ASTExpression):
pass
class ASTBooleanLiteral(ASTLiteral):
-
- def __init__(self, value: bool) ->None:
+ def __init__(self, value: bool) -> None:
self.value = value
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTBooleanLiteral):
return NotImplemented
return self.value == other.value
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.value)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.value:
+ return 'true'
+ else:
+ return 'false'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ txt = str(self)
+ signode += addnodes.desc_sig_keyword(txt, txt)
-class ASTNumberLiteral(ASTLiteral):
- def __init__(self, data: str) ->None:
+class ASTNumberLiteral(ASTLiteral):
+ def __init__(self, data: str) -> None:
self.data = data
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNumberLiteral):
return NotImplemented
return self.data == other.data
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.data)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return self.data
-class ASTCharLiteral(ASTLiteral):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ txt = str(self)
+ signode += addnodes.desc_sig_literal_number(txt, txt)
- def __init__(self, prefix: str, data: str) ->None:
- self.prefix = prefix
+
+class ASTCharLiteral(ASTLiteral):
+ def __init__(self, prefix: str, data: str) -> None:
+ self.prefix = prefix # may be None when no prefix
self.data = data
decoded = data.encode().decode('unicode-escape')
if len(decoded) == 1:
@@ -100,320 +257,626 @@ class ASTCharLiteral(ASTLiteral):
else:
raise UnsupportedMultiCharacterCharLiteral(decoded)
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTCharLiteral):
return NotImplemented
- return self.prefix == other.prefix and self.value == other.value
+ return (
+ self.prefix == other.prefix
+ and self.value == other.value
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.prefix, self.value))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.prefix is None:
+ return "'" + self.data + "'"
+ else:
+ return self.prefix + "'" + self.data + "'"
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ txt = str(self)
+ signode += addnodes.desc_sig_literal_char(txt, txt)
-class ASTStringLiteral(ASTLiteral):
- def __init__(self, data: str) ->None:
+class ASTStringLiteral(ASTLiteral):
+ def __init__(self, data: str) -> None:
self.data = data
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTStringLiteral):
return NotImplemented
return self.data == other.data
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.data)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return self.data
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ txt = str(self)
+ signode += addnodes.desc_sig_literal_string(txt, txt)
-class ASTIdExpression(ASTExpression):
- def __init__(self, name: ASTNestedName) ->None:
+class ASTIdExpression(ASTExpression):
+ def __init__(self, name: ASTNestedName) -> None:
+ # note: this class is basically to cast a nested name as an expression
self.name = name
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTIdExpression):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.name)
+
+ def get_id(self, version: int) -> str:
+ return self.name.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.name.describe_signature(signode, mode, env, symbol)
-class ASTParenExpr(ASTExpression):
- def __init__(self, expr: ASTExpression) ->None:
+class ASTParenExpr(ASTExpression):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTParenExpr):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '(' + transform(self.expr) + ')'
+
+ def get_id(self, version: int) -> str:
+ return self.expr.get_id(version) # type: ignore[attr-defined]
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.expr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+
+
+# Postfix expressions
+################################################################################
class ASTPostfixOp(ASTBase):
pass
class ASTPostfixCallExpr(ASTPostfixOp):
-
- def __init__(self, lst: (ASTParenExprList | ASTBracedInitList)) ->None:
+ def __init__(self, lst: ASTParenExprList | ASTBracedInitList) -> None:
self.lst = lst
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixCallExpr):
return NotImplemented
return self.lst == other.lst
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.lst)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.lst)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.lst.describe_signature(signode, mode, env, symbol)
-class ASTPostfixArray(ASTPostfixOp):
- def __init__(self, expr: ASTExpression) ->None:
+class ASTPostfixArray(ASTPostfixOp):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixArray):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '[' + transform(self.expr) + ']'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_punctuation('[', '[')
+ self.expr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(']', ']')
+
class ASTPostfixInc(ASTPostfixOp):
- pass
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '++'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_operator('++', '++')
class ASTPostfixDec(ASTPostfixOp):
- pass
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '--'
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_operator('--', '--')
-class ASTPostfixMemberOfPointer(ASTPostfixOp):
- def __init__(self, name: ASTNestedName) ->None:
+class ASTPostfixMemberOfPointer(ASTPostfixOp):
+ def __init__(self, name: ASTNestedName) -> None:
self.name = name
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixMemberOfPointer):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '->' + transform(self.name)
-class ASTPostfixExpr(ASTExpression):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_operator('->', '->')
+ self.name.describe_signature(signode, 'noneIsName', env, symbol)
- def __init__(self, prefix: ASTExpression, postFixes: list[ASTPostfixOp]
- ) ->None:
+
+class ASTPostfixExpr(ASTExpression):
+ def __init__(self, prefix: ASTExpression, postFixes: list[ASTPostfixOp]) -> None:
self.prefix = prefix
self.postFixes = postFixes
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixExpr):
return NotImplemented
- return (self.prefix == other.prefix and self.postFixes == other.
- postFixes)
+ return self.prefix == other.prefix and self.postFixes == other.postFixes
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.prefix, self.postFixes))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return ''.join([transform(self.prefix), *(transform(p) for p in self.postFixes)])
-class ASTUnaryOpExpr(ASTExpression):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.prefix.describe_signature(signode, mode, env, symbol)
+ for p in self.postFixes:
+ p.describe_signature(signode, mode, env, symbol)
- def __init__(self, op: str, expr: ASTExpression) ->None:
+
+# Unary expressions
+################################################################################
+
+class ASTUnaryOpExpr(ASTExpression):
+ def __init__(self, op: str, expr: ASTExpression) -> None:
self.op = op
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTUnaryOpExpr):
return NotImplemented
return self.op == other.op and self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.op, self.expr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.op[0] in 'cn':
+ return self.op + " " + transform(self.expr)
+ else:
+ return self.op + transform(self.expr)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ if self.op[0] in 'cn':
+ signode += addnodes.desc_sig_keyword(self.op, self.op)
+ signode += addnodes.desc_sig_space()
+ else:
+ signode += addnodes.desc_sig_operator(self.op, self.op)
+ self.expr.describe_signature(signode, mode, env, symbol)
-class ASTSizeofType(ASTExpression):
- def __init__(self, typ: ASTType) ->None:
+class ASTSizeofType(ASTExpression):
+ def __init__(self, typ: ASTType) -> None:
self.typ = typ
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTSizeofType):
return NotImplemented
return self.typ == other.typ
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.typ)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return "sizeof(" + transform(self.typ) + ")"
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('sizeof', 'sizeof')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.typ.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
-class ASTSizeofExpr(ASTExpression):
- def __init__(self, expr: ASTExpression) ->None:
+class ASTSizeofExpr(ASTExpression):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTSizeofExpr):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return "sizeof " + transform(self.expr)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('sizeof', 'sizeof')
+ signode += addnodes.desc_sig_space()
+ self.expr.describe_signature(signode, mode, env, symbol)
-class ASTAlignofExpr(ASTExpression):
- def __init__(self, typ: ASTType) ->None:
+class ASTAlignofExpr(ASTExpression):
+ def __init__(self, typ: ASTType) -> None:
self.typ = typ
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTAlignofExpr):
return NotImplemented
return self.typ == other.typ
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.typ)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return "alignof(" + transform(self.typ) + ")"
-class ASTCastExpr(ASTExpression):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('alignof', 'alignof')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.typ.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+
+
+# Other expressions
+################################################################################
- def __init__(self, typ: ASTType, expr: ASTExpression) ->None:
+class ASTCastExpr(ASTExpression):
+ def __init__(self, typ: ASTType, expr: ASTExpression) -> None:
self.typ = typ
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTCastExpr):
return NotImplemented
- return self.typ == other.typ and self.expr == other.expr
+ return (
+ self.typ == other.typ
+ and self.expr == other.expr
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.typ, self.expr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['(']
+ res.append(transform(self.typ))
+ res.append(')')
+ res.append(transform(self.expr))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.typ.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+ self.expr.describe_signature(signode, mode, env, symbol)
-class ASTBinOpExpr(ASTBase):
- def __init__(self, exprs: list[ASTExpression], ops: list[str]) ->None:
+class ASTBinOpExpr(ASTBase):
+ def __init__(self, exprs: list[ASTExpression], ops: list[str]) -> None:
assert len(exprs) > 0
assert len(exprs) == len(ops) + 1
self.exprs = exprs
self.ops = ops
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTBinOpExpr):
return NotImplemented
- return self.exprs == other.exprs and self.ops == other.ops
+ return (
+ self.exprs == other.exprs
+ and self.ops == other.ops
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.exprs, self.ops))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.exprs[0]))
+ for i in range(1, len(self.exprs)):
+ res.append(' ')
+ res.append(self.ops[i - 1])
+ res.append(' ')
+ res.append(transform(self.exprs[i]))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.exprs[0].describe_signature(signode, mode, env, symbol)
+ for i in range(1, len(self.exprs)):
+ signode += addnodes.desc_sig_space()
+ op = self.ops[i - 1]
+ if ord(op[0]) >= ord('a') and ord(op[0]) <= ord('z'):
+ signode += addnodes.desc_sig_keyword(op, op)
+ else:
+ signode += addnodes.desc_sig_operator(op, op)
+ signode += addnodes.desc_sig_space()
+ self.exprs[i].describe_signature(signode, mode, env, symbol)
-class ASTAssignmentExpr(ASTExpression):
- def __init__(self, exprs: list[ASTExpression], ops: list[str]) ->None:
+class ASTAssignmentExpr(ASTExpression):
+ def __init__(self, exprs: list[ASTExpression], ops: list[str]) -> None:
assert len(exprs) > 0
assert len(exprs) == len(ops) + 1
self.exprs = exprs
self.ops = ops
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTAssignmentExpr):
return NotImplemented
- return self.exprs == other.exprs and self.ops == other.ops
+ return (
+ self.exprs == other.exprs
+ and self.ops == other.ops
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.exprs, self.ops))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.exprs[0]))
+ for i in range(1, len(self.exprs)):
+ res.append(' ')
+ res.append(self.ops[i - 1])
+ res.append(' ')
+ res.append(transform(self.exprs[i]))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.exprs[0].describe_signature(signode, mode, env, symbol)
+ for i in range(1, len(self.exprs)):
+ signode += addnodes.desc_sig_space()
+ op = self.ops[i - 1]
+ if ord(op[0]) >= ord('a') and ord(op[0]) <= ord('z'):
+ signode += addnodes.desc_sig_keyword(op, op)
+ else:
+ signode += addnodes.desc_sig_operator(op, op)
+ signode += addnodes.desc_sig_space()
+ self.exprs[i].describe_signature(signode, mode, env, symbol)
-class ASTFallbackExpr(ASTExpression):
- def __init__(self, expr: str) ->None:
+class ASTFallbackExpr(ASTExpression):
+ def __init__(self, expr: str) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTFallbackExpr):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return self.expr
+
+ def get_id(self, version: int) -> str:
+ return str(self.expr)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += nodes.literal(self.expr, self.expr)
+
+
+################################################################################
+# Types
+################################################################################
class ASTTrailingTypeSpec(ASTBase):
pass
class ASTTrailingTypeSpecFundamental(ASTTrailingTypeSpec):
-
- def __init__(self, names: list[str]) ->None:
+ def __init__(self, names: list[str]) -> None:
assert len(names) != 0
self.names = names
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTrailingTypeSpecFundamental):
return NotImplemented
return self.names == other.names
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.names)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return ' '.join(self.names)
-class ASTTrailingTypeSpecName(ASTTrailingTypeSpec):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ first = True
+ for n in self.names:
+ if not first:
+ signode += addnodes.desc_sig_space()
+ else:
+ first = False
+ signode += addnodes.desc_sig_keyword_type(n, n)
- def __init__(self, prefix: str, nestedName: ASTNestedName) ->None:
+
+class ASTTrailingTypeSpecName(ASTTrailingTypeSpec):
+ def __init__(self, prefix: str, nestedName: ASTNestedName) -> None:
self.prefix = prefix
self.nestedName = nestedName
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTrailingTypeSpecName):
return NotImplemented
- return (self.prefix == other.prefix and self.nestedName == other.
- nestedName)
+ return (
+ self.prefix == other.prefix
+ and self.nestedName == other.nestedName
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.prefix, self.nestedName))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.nestedName
-class ASTFunctionParameter(ASTBase):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.prefix:
+ res.append(self.prefix)
+ res.append(' ')
+ res.append(transform(self.nestedName))
+ return ''.join(res)
- def __init__(self, arg: (ASTTypeWithInit | None), ellipsis: bool=False
- ) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ if self.prefix:
+ signode += addnodes.desc_sig_keyword(self.prefix, self.prefix)
+ signode += addnodes.desc_sig_space()
+ self.nestedName.describe_signature(signode, mode, env, symbol=symbol)
+
+
+class ASTFunctionParameter(ASTBase):
+ def __init__(self, arg: ASTTypeWithInit | None, ellipsis: bool = False) -> None:
self.arg = arg
self.ellipsis = ellipsis
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTFunctionParameter):
return NotImplemented
return self.arg == other.arg and self.ellipsis == other.ellipsis
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.arg, self.ellipsis))
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ # the anchor will be our parent
+ return symbol.parent.declaration.get_id(version, prefixed=False)
-class ASTParameters(ASTBase):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.ellipsis:
+ return '...'
+ else:
+ return transform(self.arg)
- def __init__(self, args: list[ASTFunctionParameter], attrs:
- ASTAttributeList) ->None:
+ def describe_signature(self, signode: Any, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.ellipsis:
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ else:
+ self.arg.describe_signature(signode, mode, env, symbol=symbol)
+
+
+class ASTParameters(ASTBase):
+ def __init__(self, args: list[ASTFunctionParameter], attrs: ASTAttributeList) -> None:
self.args = args
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTParameters):
return NotImplemented
return self.args == other.args and self.attrs == other.attrs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.args, self.attrs))
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.args
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append('(')
+ first = True
+ for a in self.args:
+ if not first:
+ res.append(', ')
+ first = False
+ res.append(str(a))
+ res.append(')')
+ if len(self.attrs) != 0:
+ res.append(' ')
+ res.append(transform(self.attrs))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ multi_line_parameter_list = False
+ test_node: Element = signode
+ while test_node.parent:
+ if not isinstance(test_node, addnodes.desc_signature):
+ test_node = test_node.parent
+ continue
+ multi_line_parameter_list = test_node.get('multi_line_parameter_list', False)
+ break
+
+ # only use the desc_parameterlist for the outer list, not for inner lists
+ if mode == 'lastIsName':
+ paramlist = addnodes.desc_parameterlist()
+ paramlist['multi_line_parameter_list'] = multi_line_parameter_list
+ for arg in self.args:
+ param = addnodes.desc_parameter('', '', noemph=True)
+ arg.describe_signature(param, 'param', env, symbol=symbol)
+ paramlist += param
+ signode += paramlist
+ else:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ first = True
+ for arg in self.args:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ first = False
+ arg.describe_signature(signode, 'markType', env, symbol=symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+
+ if len(self.attrs) != 0:
+ signode += addnodes.desc_sig_space()
+ self.attrs.describe_signature(signode)
-class ASTDeclSpecsSimple(ASTBaseBase):
+class ASTDeclSpecsSimple(ASTBaseBase):
def __init__(self, storage: str, threadLocal: str, inline: bool,
- restrict: bool, volatile: bool, const: bool, attrs: ASTAttributeList
- ) ->None:
+ restrict: bool, volatile: bool, const: bool, attrs: ASTAttributeList) -> None:
self.storage = storage
self.threadLocal = threadLocal
self.inline = inline
@@ -422,45 +885,159 @@ class ASTDeclSpecsSimple(ASTBaseBase):
self.const = const
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclSpecsSimple):
return NotImplemented
- return (self.storage == other.storage and self.threadLocal == other
- .threadLocal and self.inline == other.inline and self.restrict ==
- other.restrict and self.volatile == other.volatile and self.
- const == other.const and self.attrs == other.attrs)
-
- def __hash__(self) ->int:
- return hash((self.storage, self.threadLocal, self.inline, self.
- restrict, self.volatile, self.const, self.attrs))
+ return (
+ self.storage == other.storage
+ and self.threadLocal == other.threadLocal
+ and self.inline == other.inline
+ and self.restrict == other.restrict
+ and self.volatile == other.volatile
+ and self.const == other.const
+ and self.attrs == other.attrs
+ )
+
+ def __hash__(self) -> int:
+ return hash((
+ self.storage,
+ self.threadLocal,
+ self.inline,
+ self.restrict,
+ self.volatile,
+ self.const,
+ self.attrs,
+ ))
+
+ def mergeWith(self, other: ASTDeclSpecsSimple) -> ASTDeclSpecsSimple:
+ if not other:
+ return self
+ return ASTDeclSpecsSimple(self.storage or other.storage,
+ self.threadLocal or other.threadLocal,
+ self.inline or other.inline,
+ self.volatile or other.volatile,
+ self.const or other.const,
+ self.restrict or other.restrict,
+ self.attrs + other.attrs)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res: list[str] = []
+ if len(self.attrs) != 0:
+ res.append(transform(self.attrs))
+ if self.storage:
+ res.append(self.storage)
+ if self.threadLocal:
+ res.append(self.threadLocal)
+ if self.inline:
+ res.append('inline')
+ if self.restrict:
+ res.append('restrict')
+ if self.volatile:
+ res.append('volatile')
+ if self.const:
+ res.append('const')
+ return ' '.join(res)
+
+ def describe_signature(self, modifiers: list[Node]) -> None:
+ def _add(modifiers: list[Node], text: str) -> None:
+ if len(modifiers) != 0:
+ modifiers.append(addnodes.desc_sig_space())
+ modifiers.append(addnodes.desc_sig_keyword(text, text))
+
+ if len(modifiers) != 0 and len(self.attrs) != 0:
+ modifiers.append(addnodes.desc_sig_space())
+ tempNode = nodes.TextElement()
+ self.attrs.describe_signature(tempNode)
+ modifiers.extend(tempNode.children)
+ if self.storage:
+ _add(modifiers, self.storage)
+ if self.threadLocal:
+ _add(modifiers, self.threadLocal)
+ if self.inline:
+ _add(modifiers, 'inline')
+ if self.restrict:
+ _add(modifiers, 'restrict')
+ if self.volatile:
+ _add(modifiers, 'volatile')
+ if self.const:
+ _add(modifiers, 'const')
class ASTDeclSpecs(ASTBase):
-
- def __init__(self, outer: str, leftSpecs: ASTDeclSpecsSimple,
- rightSpecs: ASTDeclSpecsSimple, trailing: ASTTrailingTypeSpec) ->None:
+ def __init__(self, outer: str,
+ leftSpecs: ASTDeclSpecsSimple,
+ rightSpecs: ASTDeclSpecsSimple,
+ trailing: ASTTrailingTypeSpec) -> None:
+ # leftSpecs and rightSpecs are used for output
+ # allSpecs are used for id generation TODO: remove?
self.outer = outer
self.leftSpecs = leftSpecs
self.rightSpecs = rightSpecs
self.allSpecs = self.leftSpecs.mergeWith(self.rightSpecs)
self.trailingTypeSpec = trailing
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclSpecs):
return NotImplemented
- return (self.outer == other.outer and self.leftSpecs == other.
- leftSpecs and self.rightSpecs == other.rightSpecs and self.
- trailingTypeSpec == other.trailingTypeSpec)
-
- def __hash__(self) ->int:
- return hash((self.outer, self.leftSpecs, self.rightSpecs, self.
- trailingTypeSpec))
-
+ return (
+ self.outer == other.outer
+ and self.leftSpecs == other.leftSpecs
+ and self.rightSpecs == other.rightSpecs
+ and self.trailingTypeSpec == other.trailingTypeSpec
+ )
+
+ def __hash__(self) -> int:
+ return hash((
+ self.outer,
+ self.leftSpecs,
+ self.rightSpecs,
+ self.trailingTypeSpec,
+ ))
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res: list[str] = []
+ l = transform(self.leftSpecs)
+ if len(l) > 0:
+ res.append(l)
+ if self.trailingTypeSpec:
+ if len(res) > 0:
+ res.append(" ")
+ res.append(transform(self.trailingTypeSpec))
+ r = str(self.rightSpecs)
+ if len(r) > 0:
+ if len(res) > 0:
+ res.append(" ")
+ res.append(r)
+ return "".join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ modifiers: list[Node] = []
+
+ self.leftSpecs.describe_signature(modifiers)
+
+ for m in modifiers:
+ signode += m
+ if self.trailingTypeSpec:
+ if len(modifiers) > 0:
+ signode += addnodes.desc_sig_space()
+ self.trailingTypeSpec.describe_signature(signode, mode, env,
+ symbol=symbol)
+ modifiers = []
+ self.rightSpecs.describe_signature(modifiers)
+ if len(modifiers) > 0:
+ signode += addnodes.desc_sig_space()
+ for m in modifiers:
+ signode += m
+
+
+# Declarator
+################################################################################
class ASTArray(ASTBase):
-
- def __init__(self, static: bool, const: bool, volatile: bool, restrict:
- bool, vla: bool, size: ASTExpression) ->None:
+ def __init__(self, static: bool, const: bool, volatile: bool, restrict: bool,
+ vla: bool, size: ASTExpression) -> None:
self.static = static
self.const = const
self.volatile = volatile
@@ -472,59 +1049,182 @@ class ASTArray(ASTBase):
if size is not None:
assert not vla
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTArray):
return NotImplemented
- return (self.static == other.static and self.const == other.const and
- self.volatile == other.volatile and self.restrict == other.
- restrict and self.vla == other.vla and self.size == other.size)
-
- def __hash__(self) ->int:
- return hash((self.static, self.const, self.volatile, self.restrict,
- self.vla, self.size))
+ return (
+ self.static == other.static
+ and self.const == other.const
+ and self.volatile == other.volatile
+ and self.restrict == other.restrict
+ and self.vla == other.vla
+ and self.size == other.size
+ )
+
+ def __hash__(self) -> int:
+ return hash((
+ self.static,
+ self.const,
+ self.volatile,
+ self.restrict,
+ self.vla,
+ self.size,
+ ))
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ el = []
+ if self.static:
+ el.append('static')
+ if self.restrict:
+ el.append('restrict')
+ if self.volatile:
+ el.append('volatile')
+ if self.const:
+ el.append('const')
+ if self.vla:
+ return '[' + ' '.join(el) + '*]'
+ elif self.size:
+ el.append(transform(self.size))
+ return '[' + ' '.join(el) + ']'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('[', '[')
+ addSpace = False
+
+ def _add(signode: TextElement, text: str) -> bool:
+ if addSpace:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_keyword(text, text)
+ return True
+
+ if self.static:
+ addSpace = _add(signode, 'static')
+ if self.restrict:
+ addSpace = _add(signode, 'restrict')
+ if self.volatile:
+ addSpace = _add(signode, 'volatile')
+ if self.const:
+ addSpace = _add(signode, 'const')
+ if self.vla:
+ signode += addnodes.desc_sig_punctuation('*', '*')
+ elif self.size:
+ if addSpace:
+ signode += addnodes.desc_sig_space()
+ self.size.describe_signature(signode, 'markType', env, symbol)
+ signode += addnodes.desc_sig_punctuation(']', ']')
class ASTDeclarator(ASTBase):
- pass
+ @property
+ def name(self) -> ASTNestedName:
+ raise NotImplementedError(repr(self))
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ raise NotImplementedError(repr(self))
+
+ def require_space_after_declSpecs(self) -> bool:
+ raise NotImplementedError(repr(self))
-class ASTDeclaratorNameParam(ASTDeclarator):
- def __init__(self, declId: ASTNestedName, arrayOps: list[ASTArray],
- param: ASTParameters) ->None:
+class ASTDeclaratorNameParam(ASTDeclarator):
+ def __init__(self, declId: ASTNestedName,
+ arrayOps: list[ASTArray], param: ASTParameters) -> None:
self.declId = declId
self.arrayOps = arrayOps
self.param = param
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorNameParam):
return NotImplemented
- return (self.declId == other.declId and self.arrayOps == other.
- arrayOps and self.param == other.param)
+ return (
+ self.declId == other.declId
+ and self.arrayOps == other.arrayOps
+ and self.param == other.param
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.declId, self.arrayOps, self.param))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.declId
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.param.function_params
+
+ # ------------------------------------------------------------------------
+
+ def require_space_after_declSpecs(self) -> bool:
+ return self.declId is not None
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.declId:
+ res.append(transform(self.declId))
+ res.extend(transform(op) for op in self.arrayOps)
+ if self.param:
+ res.append(transform(self.param))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.declId:
+ self.declId.describe_signature(signode, mode, env, symbol)
+ for op in self.arrayOps:
+ op.describe_signature(signode, mode, env, symbol)
+ if self.param:
+ self.param.describe_signature(signode, mode, env, symbol)
-class ASTDeclaratorNameBitField(ASTDeclarator):
- def __init__(self, declId: ASTNestedName, size: ASTExpression) ->None:
+class ASTDeclaratorNameBitField(ASTDeclarator):
+ def __init__(self, declId: ASTNestedName, size: ASTExpression) -> None:
self.declId = declId
self.size = size
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorNameBitField):
return NotImplemented
return self.declId == other.declId and self.size == other.size
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.declId, self.size))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.declId
+
+ # ------------------------------------------------------------------------
+
+ def require_space_after_declSpecs(self) -> bool:
+ return self.declId is not None
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.declId:
+ res.append(transform(self.declId))
+ res.append(" : ")
+ res.append(transform(self.size))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.declId:
+ self.declId.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation(':', ':')
+ signode += addnodes.desc_sig_space()
+ self.size.describe_signature(signode, mode, env, symbol)
-class ASTDeclaratorPtr(ASTDeclarator):
- def __init__(self, next: ASTDeclarator, restrict: bool, volatile: bool,
- const: bool, attrs: ASTAttributeList) ->None:
+class ASTDeclaratorPtr(ASTDeclarator):
+ def __init__(self, next: ASTDeclarator, restrict: bool, volatile: bool, const: bool,
+ attrs: ASTAttributeList) -> None:
assert next
self.next = next
self.restrict = restrict
@@ -532,225 +1232,615 @@ class ASTDeclaratorPtr(ASTDeclarator):
self.const = const
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorPtr):
return NotImplemented
- return (self.next == other.next and self.restrict == other.restrict and
- self.volatile == other.volatile and self.const == other.const and
- self.attrs == other.attrs)
-
- def __hash__(self) ->int:
- return hash((self.next, self.restrict, self.volatile, self.const,
- self.attrs))
+ return (
+ self.next == other.next
+ and self.restrict == other.restrict
+ and self.volatile == other.volatile
+ and self.const == other.const
+ and self.attrs == other.attrs
+ )
+
+ def __hash__(self) -> int:
+ return hash((self.next, self.restrict, self.volatile, self.const, self.attrs))
+
+ @property
+ def name(self) -> ASTNestedName:
+ return self.next.name
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.next.function_params
+
+ def require_space_after_declSpecs(self) -> bool:
+ return self.const or self.volatile or self.restrict or \
+ len(self.attrs) > 0 or \
+ self.next.require_space_after_declSpecs()
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['*']
+ res.append(transform(self.attrs))
+ if len(self.attrs) != 0 and (self.restrict or self.volatile or self.const):
+ res.append(' ')
+ if self.restrict:
+ res.append('restrict')
+ if self.volatile:
+ if self.restrict:
+ res.append(' ')
+ res.append('volatile')
+ if self.const:
+ if self.restrict or self.volatile:
+ res.append(' ')
+ res.append('const')
+ if self.const or self.volatile or self.restrict or len(self.attrs) > 0:
+ if self.next.require_space_after_declSpecs():
+ res.append(' ')
+ res.append(transform(self.next))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('*', '*')
+ self.attrs.describe_signature(signode)
+ if len(self.attrs) != 0 and (self.restrict or self.volatile or self.const):
+ signode += addnodes.desc_sig_space()
+
+ def _add_anno(signode: TextElement, text: str) -> None:
+ signode += addnodes.desc_sig_keyword(text, text)
+
+ if self.restrict:
+ _add_anno(signode, 'restrict')
+ if self.volatile:
+ if self.restrict:
+ signode += addnodes.desc_sig_space()
+ _add_anno(signode, 'volatile')
+ if self.const:
+ if self.restrict or self.volatile:
+ signode += addnodes.desc_sig_space()
+ _add_anno(signode, 'const')
+ if self.const or self.volatile or self.restrict or len(self.attrs) > 0:
+ if self.next.require_space_after_declSpecs():
+ signode += addnodes.desc_sig_space()
+ self.next.describe_signature(signode, mode, env, symbol)
class ASTDeclaratorParen(ASTDeclarator):
-
- def __init__(self, inner: ASTDeclarator, next: ASTDeclarator) ->None:
+ def __init__(self, inner: ASTDeclarator, next: ASTDeclarator) -> None:
assert inner
assert next
self.inner = inner
self.next = next
+ # TODO: we assume the name and params are in inner
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorParen):
return NotImplemented
return self.inner == other.inner and self.next == other.next
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.inner, self.next))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.inner.name
-class ASTParenExprList(ASTBaseParenExprList):
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.inner.function_params
+
+ def require_space_after_declSpecs(self) -> bool:
+ return True
- def __init__(self, exprs: list[ASTExpression]) ->None:
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['(']
+ res.append(transform(self.inner))
+ res.append(')')
+ res.append(transform(self.next))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.inner.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+ self.next.describe_signature(signode, "noneIsName", env, symbol)
+
+
+# Initializer
+################################################################################
+
+class ASTParenExprList(ASTBaseParenExprList):
+ def __init__(self, exprs: list[ASTExpression]) -> None:
self.exprs = exprs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTParenExprList):
return NotImplemented
return self.exprs == other.exprs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.exprs)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ exprs = [transform(e) for e in self.exprs]
+ return '(%s)' % ', '.join(exprs)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ first = True
+ for e in self.exprs:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ else:
+ first = False
+ e.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
-class ASTBracedInitList(ASTBase):
- def __init__(self, exprs: list[ASTExpression], trailingComma: bool) ->None:
+class ASTBracedInitList(ASTBase):
+ def __init__(self, exprs: list[ASTExpression], trailingComma: bool) -> None:
self.exprs = exprs
self.trailingComma = trailingComma
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTBracedInitList):
return NotImplemented
- return (self.exprs == other.exprs and self.trailingComma == other.
- trailingComma)
+ return self.exprs == other.exprs and self.trailingComma == other.trailingComma
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.exprs, self.trailingComma))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ exprs = ', '.join(transform(e) for e in self.exprs)
+ trailingComma = ',' if self.trailingComma else ''
+ return f'{{{exprs}{trailingComma}}}'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('{', '{')
+ first = True
+ for e in self.exprs:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ else:
+ first = False
+ e.describe_signature(signode, mode, env, symbol)
+ if self.trailingComma:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_punctuation('}', '}')
-class ASTInitializer(ASTBase):
- def __init__(self, value: (ASTBracedInitList | ASTExpression),
- hasAssign: bool=True) ->None:
+class ASTInitializer(ASTBase):
+ def __init__(self, value: ASTBracedInitList | ASTExpression,
+ hasAssign: bool = True) -> None:
self.value = value
self.hasAssign = hasAssign
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTInitializer):
return NotImplemented
return self.value == other.value and self.hasAssign == other.hasAssign
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.value, self.hasAssign))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ val = transform(self.value)
+ if self.hasAssign:
+ return ' = ' + val
+ else:
+ return val
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.hasAssign:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation('=', '=')
+ signode += addnodes.desc_sig_space()
+ self.value.describe_signature(signode, 'markType', env, symbol)
-class ASTType(ASTBase):
- def __init__(self, declSpecs: ASTDeclSpecs, decl: ASTDeclarator) ->None:
+class ASTType(ASTBase):
+ def __init__(self, declSpecs: ASTDeclSpecs, decl: ASTDeclarator) -> None:
assert declSpecs
assert decl
self.declSpecs = declSpecs
self.decl = decl
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTType):
return NotImplemented
return self.declSpecs == other.declSpecs and self.decl == other.decl
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.declSpecs, self.decl))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.decl.name
+
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ return symbol.get_full_nested_name().get_id(version)
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.decl.function_params
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ declSpecs = transform(self.declSpecs)
+ res.append(declSpecs)
+ if self.decl.require_space_after_declSpecs() and len(declSpecs) > 0:
+ res.append(' ')
+ res.append(transform(self.decl))
+ return ''.join(res)
+
+ def get_type_declaration_prefix(self) -> str:
+ if self.declSpecs.trailingTypeSpec:
+ return 'typedef'
+ else:
+ return 'type'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.declSpecs.describe_signature(signode, 'markType', env, symbol)
+ if (self.decl.require_space_after_declSpecs() and
+ len(str(self.declSpecs)) > 0):
+ signode += addnodes.desc_sig_space()
+ # for parameters that don't really declare new names we get 'markType',
+ # this should not be propagated, but be 'noneIsName'.
+ if mode == 'markType':
+ mode = 'noneIsName'
+ self.decl.describe_signature(signode, mode, env, symbol)
-class ASTTypeWithInit(ASTBase):
- def __init__(self, type: ASTType, init: ASTInitializer) ->None:
+class ASTTypeWithInit(ASTBase):
+ def __init__(self, type: ASTType, init: ASTInitializer) -> None:
self.type = type
self.init = init
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTypeWithInit):
return NotImplemented
return self.type == other.type and self.init == other.init
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.type, self.init))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.type.name
-class ASTMacroParameter(ASTBase):
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ return self.type.get_id(version, objectType, symbol)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.type))
+ if self.init:
+ res.append(transform(self.init))
+ return ''.join(res)
- def __init__(self, arg: (ASTNestedName | None), ellipsis: bool=False,
- variadic: bool=False) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.type.describe_signature(signode, mode, env, symbol)
+ if self.init:
+ self.init.describe_signature(signode, mode, env, symbol)
+
+
+class ASTMacroParameter(ASTBase):
+ def __init__(self, arg: ASTNestedName | None, ellipsis: bool = False,
+ variadic: bool = False) -> None:
self.arg = arg
self.ellipsis = ellipsis
self.variadic = variadic
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTMacroParameter):
return NotImplemented
- return (self.arg == other.arg and self.ellipsis == other.ellipsis and
- self.variadic == other.variadic)
+ return (
+ self.arg == other.arg
+ and self.ellipsis == other.ellipsis
+ and self.variadic == other.variadic
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.arg, self.ellipsis, self.variadic))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.ellipsis:
+ return '...'
+ elif self.variadic:
+ return transform(self.arg) + '...'
+ else:
+ return transform(self.arg)
+
+ def describe_signature(self, signode: Any, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.ellipsis:
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ elif self.variadic:
+ name = str(self)
+ signode += addnodes.desc_sig_name(name, name)
+ else:
+ self.arg.describe_signature(signode, mode, env, symbol=symbol)
+
class ASTMacro(ASTBase):
-
- def __init__(self, ident: ASTNestedName, args: (list[ASTMacroParameter] |
- None)) ->None:
+ def __init__(self, ident: ASTNestedName, args: list[ASTMacroParameter] | None) -> None:
self.ident = ident
self.args = args
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTMacro):
return NotImplemented
return self.ident == other.ident and self.args == other.args
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.ident, self.args))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.ident
+
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.ident))
+ if self.args is not None:
+ res.append('(')
+ first = True
+ for arg in self.args:
+ if not first:
+ res.append(', ')
+ first = False
+ res.append(transform(arg))
+ res.append(')')
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.ident.describe_signature(signode, mode, env, symbol)
+ if self.args is None:
+ return
+ paramlist = addnodes.desc_parameterlist()
+ for arg in self.args:
+ param = addnodes.desc_parameter('', '', noemph=True)
+ arg.describe_signature(param, 'param', env, symbol=symbol)
+ paramlist += param
+ signode += paramlist
-class ASTStruct(ASTBase):
- def __init__(self, name: ASTNestedName) ->None:
+class ASTStruct(ASTBase):
+ def __init__(self, name: ASTNestedName) -> None:
self.name = name
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTStruct):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ return symbol.get_full_nested_name().get_id(version)
-class ASTUnion(ASTBase):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.name)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.name.describe_signature(signode, mode, env, symbol=symbol)
- def __init__(self, name: ASTNestedName) ->None:
+
+class ASTUnion(ASTBase):
+ def __init__(self, name: ASTNestedName) -> None:
self.name = name
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTUnion):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ return symbol.get_full_nested_name().get_id(version)
-class ASTEnum(ASTBase):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.name)
- def __init__(self, name: ASTNestedName) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.name.describe_signature(signode, mode, env, symbol=symbol)
+
+
+class ASTEnum(ASTBase):
+ def __init__(self, name: ASTNestedName) -> None:
self.name = name
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTEnum):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.name)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.name.describe_signature(signode, mode, env, symbol=symbol)
-class ASTEnumerator(ASTBase):
- def __init__(self, name: ASTNestedName, init: (ASTInitializer | None),
- attrs: ASTAttributeList) ->None:
+class ASTEnumerator(ASTBase):
+ def __init__(self, name: ASTNestedName, init: ASTInitializer | None,
+ attrs: ASTAttributeList) -> None:
self.name = name
self.init = init
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTEnumerator):
return NotImplemented
- return (self.name == other.name and self.init == other.init and
- self.attrs == other.attrs)
+ return (
+ self.name == other.name
+ and self.init == other.init
+ and self.attrs == other.attrs
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.name, self.init, self.attrs))
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.name))
+ if len(self.attrs) != 0:
+ res.append(' ')
+ res.append(transform(self.attrs))
+ if self.init:
+ res.append(transform(self.init))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.name.describe_signature(signode, mode, env, symbol)
+ if len(self.attrs) != 0:
+ signode += addnodes.desc_sig_space()
+ self.attrs.describe_signature(signode)
+ if self.init:
+ self.init.describe_signature(signode, 'markType', env, symbol)
-class ASTDeclaration(ASTBaseBase):
- def __init__(self, objectType: str, directiveType: (str | None),
- declaration: (DeclarationType | ASTFunctionParameter), semicolon:
- bool=False) ->None:
+class ASTDeclaration(ASTBaseBase):
+ def __init__(self, objectType: str, directiveType: str | None,
+ declaration: DeclarationType | ASTFunctionParameter,
+ semicolon: bool = False) -> None:
self.objectType = objectType
self.directiveType = directiveType
self.declaration = declaration
self.semicolon = semicolon
+
self.symbol: Symbol | None = None
+ # set by CObject._add_enumerator_to_parent
self.enumeratorScopedSymbol: Symbol | None = None
+
+ # the cache assumes that by the time get_newest_id is called, no
+ # further changes will be made to this object
self._newest_id_cache: str | None = None
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaration):
return NotImplemented
- return (self.objectType == other.objectType and self.directiveType ==
- other.directiveType and self.declaration == other.declaration and
- self.semicolon == other.semicolon and self.symbol == other.
- symbol and self.enumeratorScopedSymbol == other.
- enumeratorScopedSymbol)
+ return (
+ self.objectType == other.objectType
+ and self.directiveType == other.directiveType
+ and self.declaration == other.declaration
+ and self.semicolon == other.semicolon
+ and self.symbol == other.symbol
+ and self.enumeratorScopedSymbol == other.enumeratorScopedSymbol
+ )
+
+ def clone(self) -> ASTDeclaration:
+ return ASTDeclaration(self.objectType, self.directiveType,
+ self.declaration.clone(), self.semicolon)
+
+ @property
+ def name(self) -> ASTNestedName:
+ decl = cast(DeclarationType, self.declaration)
+ return decl.name
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter] | None:
+ if self.objectType != 'function':
+ return None
+ decl = cast(ASTType, self.declaration)
+ return decl.function_params
+
+ def get_id(self, version: int, prefixed: bool = True) -> str:
+ if self.objectType == 'enumerator' and self.enumeratorScopedSymbol:
+ return self.enumeratorScopedSymbol.declaration.get_id(version, prefixed)
+ id_ = self.declaration.get_id(version, self.objectType, self.symbol)
+ if prefixed:
+ return _id_prefix[version] + id_
+ else:
+ return id_
+
+ def get_newest_id(self) -> str:
+ if self._newest_id_cache is None:
+ self._newest_id_cache = self.get_id(_max_id, True)
+ return self._newest_id_cache
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = transform(self.declaration)
+ if self.semicolon:
+ res += ';'
+ return res
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, options: dict[str, bool]) -> None:
+ verify_description_mode(mode)
+ assert self.symbol
+ # The caller of the domain added a desc_signature node.
+ # Always enable multiline:
+ signode['is_multiline'] = True
+ # Put each line in a desc_signature_line node.
+ mainDeclNode = addnodes.desc_signature_line()
+ mainDeclNode.sphinx_line_type = 'declarator'
+ mainDeclNode['add_permalink'] = not self.symbol.isRedeclaration
+ signode += mainDeclNode
+
+ if self.objectType in {'member', 'function', 'macro'}:
+ pass
+ elif self.objectType == 'struct':
+ mainDeclNode += addnodes.desc_sig_keyword('struct', 'struct')
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.objectType == 'union':
+ mainDeclNode += addnodes.desc_sig_keyword('union', 'union')
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.objectType == 'enum':
+ mainDeclNode += addnodes.desc_sig_keyword('enum', 'enum')
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.objectType == 'enumerator':
+ mainDeclNode += addnodes.desc_sig_keyword('enumerator', 'enumerator')
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.objectType == 'type':
+ decl = cast(ASTType, self.declaration)
+ prefix = decl.get_type_declaration_prefix()
+ mainDeclNode += addnodes.desc_sig_keyword(prefix, prefix)
+ mainDeclNode += addnodes.desc_sig_space()
+ else:
+ raise AssertionError
+ self.declaration.describe_signature(mainDeclNode, mode, env, self.symbol)
+ if self.semicolon:
+ mainDeclNode += addnodes.desc_sig_punctuation(';', ';')
diff --git a/sphinx/domains/c/_ids.py b/sphinx/domains/c/_ids.py
index 76036e7d2..cd617be90 100644
--- a/sphinx/domains/c/_ids.py
+++ b/sphinx/domains/c/_ids.py
@@ -1,30 +1,53 @@
from __future__ import annotations
+
import re
-_keywords = ['auto', 'break', 'case', 'char', 'const', 'continue',
- 'default', 'do', 'double', 'else', 'enum', 'extern', 'float', 'for',
- 'goto', 'if', 'inline', 'int', 'long', 'register', 'restrict', 'return',
- 'short', 'signed', 'sizeof', 'static', 'struct', 'switch', 'typedef',
- 'union', 'unsigned', 'void', 'volatile', 'while', '_Alignas',
- '_Alignof', '_Atomic', '_Bool', '_Complex', '_Decimal32', '_Decimal64',
- '_Decimal128', '_Generic', '_Imaginary', '_Noreturn', '_Static_assert',
- '_Thread_local']
-_macroKeywords = ['alignas', 'alignof', 'bool', 'complex', 'imaginary',
- 'noreturn', 'static_assert', 'thread_local']
-_expression_bin_ops = [['||', 'or'], ['&&', 'and'], ['|', 'bitor'], ['^',
- 'xor'], ['&', 'bitand'], ['==', '!=', 'not_eq'], ['<=', '>=', '<', '>'],
- ['<<', '>>'], ['+', '-'], ['*', '/', '%'], ['.*', '->*']]
-_expression_unary_ops = ['++', '--', '*', '&', '+', '-', '!', 'not', '~',
- 'compl']
-_expression_assignment_ops = ['=', '*=', '/=', '%=', '+=', '-=', '>>=',
- '<<=', '&=', 'and_eq', '^=', 'xor_eq', '|=', 'or_eq']
+
+# https://en.cppreference.com/w/c/keyword
+_keywords = [
+ 'auto', 'break', 'case', 'char', 'const', 'continue', 'default', 'do', 'double',
+ 'else', 'enum', 'extern', 'float', 'for', 'goto', 'if', 'inline', 'int', 'long',
+ 'register', 'restrict', 'return', 'short', 'signed', 'sizeof', 'static', 'struct',
+ 'switch', 'typedef', 'union', 'unsigned', 'void', 'volatile', 'while',
+ '_Alignas', '_Alignof', '_Atomic', '_Bool', '_Complex',
+ '_Decimal32', '_Decimal64', '_Decimal128',
+ '_Generic', '_Imaginary', '_Noreturn', '_Static_assert', '_Thread_local',
+]
+# These are only keyword'y when the corresponding headers are included.
+# They are used as default value for c_extra_keywords.
+_macroKeywords = [
+ 'alignas', 'alignof', 'bool', 'complex', 'imaginary', 'noreturn', 'static_assert',
+ 'thread_local',
+]
+
+# these are ordered by precedence
+_expression_bin_ops = [
+ ['||', 'or'],
+ ['&&', 'and'],
+ ['|', 'bitor'],
+ ['^', 'xor'],
+ ['&', 'bitand'],
+ ['==', '!=', 'not_eq'],
+ ['<=', '>=', '<', '>'],
+ ['<<', '>>'],
+ ['+', '-'],
+ ['*', '/', '%'],
+ ['.*', '->*'],
+]
+_expression_unary_ops = ["++", "--", "*", "&", "+", "-", "!", "not", "~", "compl"]
+_expression_assignment_ops = ["=", "*=", "/=", "%=", "+=", "-=",
+ ">>=", "<<=", "&=", "and_eq", "^=", "xor_eq", "|=", "or_eq"]
+
_max_id = 1
_id_prefix = [None, 'c.', 'Cv2.']
-_string_re = re.compile(
- '[LuU8]?(\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'|"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)")'
- , re.DOTALL)
-_simple_type_specifiers_re = re.compile(
- """
- \\b(
+# Ids are used in lookup keys which are used across pickled files,
+# so when _max_id changes, make sure to update the ENV_VERSION.
+
+_string_re = re.compile(r"[LuU8]?('([^'\\]*(?:\\.[^'\\]*)*)'"
+ r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.DOTALL)
+
+# bool, complex, and imaginary are macro "keywords", so they are handled separately
+_simple_type_specifiers_re = re.compile(r"""
+ \b(
void|_Bool
|signed|unsigned
|short|long
@@ -38,6 +61,5 @@ _simple_type_specifiers_re = re.compile(
|__float80|_Float64x|__float128|_Float128|__ibm128 # extension
|__fp16 # extension
|_Sat|_Fract|fract|_Accum|accum # extension
- )\\b
-"""
- , re.VERBOSE)
+ )\b
+""", re.VERBOSE)
diff --git a/sphinx/domains/c/_parser.py b/sphinx/domains/c/_parser.py
index c92013687..1d29c6083 100644
--- a/sphinx/domains/c/_parser.py
+++ b/sphinx/domains/c/_parser.py
@@ -1,24 +1,1053 @@
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any
-from sphinx.domains.c._ast import ASTAlignofExpr, ASTArray, ASTAssignmentExpr, ASTBinOpExpr, ASTBooleanLiteral, ASTBracedInitList, ASTCastExpr, ASTCharLiteral, ASTDeclaration, ASTDeclarator, ASTDeclaratorNameBitField, ASTDeclaratorNameParam, ASTDeclaratorParen, ASTDeclaratorPtr, ASTDeclSpecs, ASTDeclSpecsSimple, ASTEnum, ASTEnumerator, ASTExpression, ASTFallbackExpr, ASTFunctionParameter, ASTIdentifier, ASTIdExpression, ASTInitializer, ASTLiteral, ASTMacro, ASTMacroParameter, ASTNestedName, ASTNumberLiteral, ASTParameters, ASTParenExpr, ASTParenExprList, ASTPostfixArray, ASTPostfixCallExpr, ASTPostfixDec, ASTPostfixExpr, ASTPostfixInc, ASTPostfixMemberOfPointer, ASTPostfixOp, ASTSizeofExpr, ASTSizeofType, ASTStringLiteral, ASTStruct, ASTTrailingTypeSpec, ASTTrailingTypeSpecFundamental, ASTTrailingTypeSpecName, ASTType, ASTTypeWithInit, ASTUnaryOpExpr, ASTUnion
-from sphinx.domains.c._ids import _expression_assignment_ops, _expression_bin_ops, _expression_unary_ops, _keywords, _simple_type_specifiers_re, _string_re
-from sphinx.util.cfamily import ASTAttributeList, BaseParser, DefinitionError, UnsupportedMultiCharacterCharLiteral, binary_literal_re, char_literal_re, float_literal_re, float_literal_suffix_re, hex_literal_re, identifier_re, integer_literal_re, integers_literal_suffix_re, octal_literal_re
+
+from sphinx.domains.c._ast import (
+ ASTAlignofExpr,
+ ASTArray,
+ ASTAssignmentExpr,
+ ASTBinOpExpr,
+ ASTBooleanLiteral,
+ ASTBracedInitList,
+ ASTCastExpr,
+ ASTCharLiteral,
+ ASTDeclaration,
+ ASTDeclarator,
+ ASTDeclaratorNameBitField,
+ ASTDeclaratorNameParam,
+ ASTDeclaratorParen,
+ ASTDeclaratorPtr,
+ ASTDeclSpecs,
+ ASTDeclSpecsSimple,
+ ASTEnum,
+ ASTEnumerator,
+ ASTExpression,
+ ASTFallbackExpr,
+ ASTFunctionParameter,
+ ASTIdentifier,
+ ASTIdExpression,
+ ASTInitializer,
+ ASTLiteral,
+ ASTMacro,
+ ASTMacroParameter,
+ ASTNestedName,
+ ASTNumberLiteral,
+ ASTParameters,
+ ASTParenExpr,
+ ASTParenExprList,
+ ASTPostfixArray,
+ ASTPostfixCallExpr,
+ ASTPostfixDec,
+ ASTPostfixExpr,
+ ASTPostfixInc,
+ ASTPostfixMemberOfPointer,
+ ASTPostfixOp,
+ ASTSizeofExpr,
+ ASTSizeofType,
+ ASTStringLiteral,
+ ASTStruct,
+ ASTTrailingTypeSpec,
+ ASTTrailingTypeSpecFundamental,
+ ASTTrailingTypeSpecName,
+ ASTType,
+ ASTTypeWithInit,
+ ASTUnaryOpExpr,
+ ASTUnion,
+)
+from sphinx.domains.c._ids import (
+ _expression_assignment_ops,
+ _expression_bin_ops,
+ _expression_unary_ops,
+ _keywords,
+ _simple_type_specifiers_re,
+ _string_re,
+)
+from sphinx.util.cfamily import (
+ ASTAttributeList,
+ BaseParser,
+ DefinitionError,
+ UnsupportedMultiCharacterCharLiteral,
+ binary_literal_re,
+ char_literal_re,
+ float_literal_re,
+ float_literal_suffix_re,
+ hex_literal_re,
+ identifier_re,
+ integer_literal_re,
+ integers_literal_suffix_re,
+ octal_literal_re,
+)
+
if TYPE_CHECKING:
from collections.abc import Callable, Sequence
+
from sphinx.domains.c._ast import DeclarationType
class DefinitionParser(BaseParser):
+ @property
+ def language(self) -> str:
+ return 'C'
+
+ @property
+ def id_attributes(self) -> Sequence[str]:
+ return self.config.c_id_attributes
+
+ @property
+ def paren_attributes(self) -> Sequence[str]:
+ return self.config.c_paren_attributes
+
+ def _parse_string(self) -> str | None:
+ if self.current_char != '"':
+ return None
+ startPos = self.pos
+ self.pos += 1
+ escape = False
+ while True:
+ if self.eof:
+ self.fail("Unexpected end during inside string.")
+ elif self.current_char == '"' and not escape:
+ self.pos += 1
+ break
+ elif self.current_char == '\\':
+ escape = True
+ else:
+ escape = False
+ self.pos += 1
+ return self.definition[startPos:self.pos]
+
+ def _parse_literal(self) -> ASTLiteral | None:
+ # -> integer-literal
+ # | character-literal
+ # | floating-literal
+ # | string-literal
+ # | boolean-literal -> "false" | "true"
+ self.skip_ws()
+ if self.skip_word('true'):
+ return ASTBooleanLiteral(True)
+ if self.skip_word('false'):
+ return ASTBooleanLiteral(False)
+ pos = self.pos
+ if self.match(float_literal_re):
+ self.match(float_literal_suffix_re)
+ return ASTNumberLiteral(self.definition[pos:self.pos])
+ for regex in (binary_literal_re, hex_literal_re,
+ integer_literal_re, octal_literal_re):
+ if self.match(regex):
+ self.match(integers_literal_suffix_re)
+ return ASTNumberLiteral(self.definition[pos:self.pos])
+
+ string = self._parse_string()
+ if string is not None:
+ return ASTStringLiteral(string)
+
+ # character-literal
+ if self.match(char_literal_re):
+ prefix = self.last_match.group(1) # may be None when no prefix
+ data = self.last_match.group(2)
+ try:
+ return ASTCharLiteral(prefix, data)
+ except UnicodeDecodeError as e:
+ self.fail("Can not handle character literal. Internal error was: %s" % e)
+ except UnsupportedMultiCharacterCharLiteral:
+ self.fail("Can not handle character literal"
+ " resulting in multiple decoded characters.")
+ return None
+
+ def _parse_paren_expression(self) -> ASTExpression | None:
+ # "(" expression ")"
+ if self.current_char != '(':
+ return None
+ self.pos += 1
+ res = self._parse_expression()
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expected ')' in end of parenthesized expression.")
+ return ASTParenExpr(res)
+
+ def _parse_primary_expression(self) -> ASTExpression | None:
+ # literal
+ # "(" expression ")"
+ # id-expression -> we parse this with _parse_nested_name
+ self.skip_ws()
+ res: ASTExpression | None = self._parse_literal()
+ if res is not None:
+ return res
+ res = self._parse_paren_expression()
+ if res is not None:
+ return res
+ nn = self._parse_nested_name()
+ if nn is not None:
+ return ASTIdExpression(nn)
+ return None
+
+ def _parse_initializer_list(self, name: str, open: str, close: str,
+ ) -> tuple[list[ASTExpression] | None, bool | None]:
+ # Parse open and close with the actual initializer-list in between
+ # -> initializer-clause '...'[opt]
+ # | initializer-list ',' initializer-clause '...'[opt]
+ # TODO: designators
+ self.skip_ws()
+ if not self.skip_string_and_ws(open):
+ return None, None
+ if self.skip_string(close):
+ return [], False
+
+ exprs = []
+ trailingComma = False
+ while True:
+ self.skip_ws()
+ expr = self._parse_expression()
+ self.skip_ws()
+ exprs.append(expr)
+ self.skip_ws()
+ if self.skip_string(close):
+ break
+ if not self.skip_string_and_ws(','):
+ self.fail(f"Error in {name}, expected ',' or '{close}'.")
+ if self.current_char == close == '}':
+ self.pos += 1
+ trailingComma = True
+ break
+ return exprs, trailingComma
+
+ def _parse_paren_expression_list(self) -> ASTParenExprList | None:
+ # -> '(' expression-list ')'
+ # though, we relax it to also allow empty parens
+ # as it's needed in some cases
+ #
+ # expression-list
+ # -> initializer-list
+ exprs, trailingComma = self._parse_initializer_list("parenthesized expression-list",
+ '(', ')')
+ if exprs is None:
+ return None
+ return ASTParenExprList(exprs)
+
+ def _parse_braced_init_list(self) -> ASTBracedInitList | None:
+ # -> '{' initializer-list ','[opt] '}'
+ # | '{' '}'
+ exprs, trailingComma = self._parse_initializer_list("braced-init-list", '{', '}')
+ if exprs is None:
+ return None
+ return ASTBracedInitList(exprs, trailingComma)
+
+ def _parse_postfix_expression(self) -> ASTPostfixExpr:
+ # -> primary
+ # | postfix "[" expression "]"
+ # | postfix "[" braced-init-list [opt] "]"
+ # | postfix "(" expression-list [opt] ")"
+ # | postfix "." id-expression // taken care of in primary by nested name
+ # | postfix "->" id-expression
+ # | postfix "++"
+ # | postfix "--"
+
+ prefix = self._parse_primary_expression()
+
+ # and now parse postfixes
+ postFixes: list[ASTPostfixOp] = []
+ while True:
+ self.skip_ws()
+ if self.skip_string_and_ws('['):
+ expr = self._parse_expression()
+ self.skip_ws()
+ if not self.skip_string(']'):
+ self.fail("Expected ']' in end of postfix expression.")
+ postFixes.append(ASTPostfixArray(expr))
+ continue
+ if self.skip_string('->'):
+ if self.skip_string('*'):
+ # don't steal the arrow
+ self.pos -= 3
+ else:
+ name = self._parse_nested_name()
+ postFixes.append(ASTPostfixMemberOfPointer(name))
+ continue
+ if self.skip_string('++'):
+ postFixes.append(ASTPostfixInc())
+ continue
+ if self.skip_string('--'):
+ postFixes.append(ASTPostfixDec())
+ continue
+ lst = self._parse_paren_expression_list()
+ if lst is not None:
+ postFixes.append(ASTPostfixCallExpr(lst))
+ continue
+ break
+ return ASTPostfixExpr(prefix, postFixes)
+
+ def _parse_unary_expression(self) -> ASTExpression:
+ # -> postfix
+ # | "++" cast
+ # | "--" cast
+ # | unary-operator cast -> (* | & | + | - | ! | ~) cast
+ # The rest:
+ # | "sizeof" unary
+ # | "sizeof" "(" type-id ")"
+ # | "alignof" "(" type-id ")"
+ self.skip_ws()
+ for op in _expression_unary_ops:
+ # TODO: hmm, should we be able to backtrack here?
+ if op[0] in 'cn':
+ res = self.skip_word(op)
+ else:
+ res = self.skip_string(op)
+ if res:
+ expr = self._parse_cast_expression()
+ return ASTUnaryOpExpr(op, expr)
+ if self.skip_word_and_ws('sizeof'):
+ if self.skip_string_and_ws('('):
+ typ = self._parse_type(named=False)
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expecting ')' to end 'sizeof'.")
+ return ASTSizeofType(typ)
+ expr = self._parse_unary_expression()
+ return ASTSizeofExpr(expr)
+ if self.skip_word_and_ws('alignof'):
+ if not self.skip_string_and_ws('('):
+ self.fail("Expecting '(' after 'alignof'.")
+ typ = self._parse_type(named=False)
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expecting ')' to end 'alignof'.")
+ return ASTAlignofExpr(typ)
+ return self._parse_postfix_expression()
+
+ def _parse_cast_expression(self) -> ASTExpression:
+ # -> unary | "(" type-id ")" cast
+ pos = self.pos
+ self.skip_ws()
+ if self.skip_string('('):
+ try:
+ typ = self._parse_type(False)
+ if not self.skip_string(')'):
+ self.fail("Expected ')' in cast expression.")
+ expr = self._parse_cast_expression()
+ return ASTCastExpr(typ, expr)
+ except DefinitionError as exCast:
+ self.pos = pos
+ try:
+ return self._parse_unary_expression()
+ except DefinitionError as exUnary:
+ errs = []
+ errs.append((exCast, "If type cast expression"))
+ errs.append((exUnary, "If unary expression"))
+ raise self._make_multi_error(errs,
+ "Error in cast expression.") from exUnary
+ else:
+ return self._parse_unary_expression()
+
+ def _parse_logical_or_expression(self) -> ASTExpression:
+ # logical-or = logical-and ||
+ # logical-and = inclusive-or &&
+ # inclusive-or = exclusive-or |
+ # exclusive-or = and ^
+ # and = equality &
+ # equality = relational ==, !=
+ # relational = shift <, >, <=, >=
+ # shift = additive <<, >>
+ # additive = multiplicative +, -
+ # multiplicative = pm *, /, %
+ # pm = cast .*, ->*
+ def _parse_bin_op_expr(self: DefinitionParser, opId: int) -> ASTExpression:
+ if opId + 1 == len(_expression_bin_ops):
+ def parser() -> ASTExpression:
+ return self._parse_cast_expression()
+ else:
+ def parser() -> ASTExpression:
+ return _parse_bin_op_expr(self, opId + 1)
+ exprs = []
+ ops = []
+ exprs.append(parser())
+ while True:
+ self.skip_ws()
+ pos = self.pos
+ oneMore = False
+ for op in _expression_bin_ops[opId]:
+ if op[0] in 'abcnox':
+ if not self.skip_word(op):
+ continue
+ else:
+ if not self.skip_string(op):
+ continue
+ if op == self.current_char == '&':
+ # don't split the && 'token'
+ self.pos -= 1
+ # and btw. && has lower precedence, so we are done
+ break
+ try:
+ expr = parser()
+ exprs.append(expr)
+ ops.append(op)
+ oneMore = True
+ break
+ except DefinitionError:
+ self.pos = pos
+ if not oneMore:
+ break
+ return ASTBinOpExpr(exprs, ops) # type: ignore[return-value]
+ return _parse_bin_op_expr(self, 0)
+
+ def _parse_conditional_expression_tail(self, orExprHead: Any) -> ASTExpression | None:
+ # -> "?" expression ":" assignment-expression
+ return None
+
+ def _parse_assignment_expression(self) -> ASTExpression:
+ # -> conditional-expression
+ # | logical-or-expression assignment-operator initializer-clause
+ # -> conditional-expression ->
+ # logical-or-expression
+ # | logical-or-expression "?" expression ":" assignment-expression
+ # | logical-or-expression assignment-operator initializer-clause
+ exprs = []
+ ops = []
+ orExpr = self._parse_logical_or_expression()
+ exprs.append(orExpr)
+ # TODO: handle ternary with _parse_conditional_expression_tail
+ while True:
+ oneMore = False
+ self.skip_ws()
+ for op in _expression_assignment_ops:
+ if op[0] in 'abcnox':
+ if not self.skip_word(op):
+ continue
+ else:
+ if not self.skip_string(op):
+ continue
+ expr = self._parse_logical_or_expression()
+ exprs.append(expr)
+ ops.append(op)
+ oneMore = True
+ if not oneMore:
+ break
+ return ASTAssignmentExpr(exprs, ops)
+
+ def _parse_constant_expression(self) -> ASTExpression:
+ # -> conditional-expression
+ orExpr = self._parse_logical_or_expression()
+ # TODO: use _parse_conditional_expression_tail
+ return orExpr
+
+ def _parse_expression(self) -> ASTExpression:
+ # -> assignment-expression
+ # | expression "," assignment-expression
+ # TODO: actually parse the second production
+ return self._parse_assignment_expression()
+
+ def _parse_expression_fallback(
+ self, end: list[str],
+ parser: Callable[[], ASTExpression],
+ allow: bool = True) -> ASTExpression:
+ # Stupidly "parse" an expression.
+ # 'end' should be a list of characters which ends the expression.
+
+ # first try to use the provided parser
+ prevPos = self.pos
+ try:
+ return parser()
+ except DefinitionError as e:
+ # some places (e.g., template parameters) we really don't want to use fallback,
+ # and for testing we may want to globally disable it
+ if not allow or not self.allowFallbackExpressionParsing:
+ raise
+ self.warn("Parsing of expression failed. Using fallback parser."
+ " Error was:\n%s" % e)
+ self.pos = prevPos
+ # and then the fallback scanning
+ assert end is not None
+ self.skip_ws()
+ startPos = self.pos
+ if self.match(_string_re):
+ value = self.matched_text
+ else:
+ # TODO: add handling of more bracket-like things, and quote handling
+ brackets = {'(': ')', '{': '}', '[': ']'}
+ symbols: list[str] = []
+ while not self.eof:
+ if (len(symbols) == 0 and self.current_char in end):
+ break
+ if self.current_char in brackets:
+ symbols.append(brackets[self.current_char])
+ elif len(symbols) > 0 and self.current_char == symbols[-1]:
+ symbols.pop()
+ self.pos += 1
+ if len(end) > 0 and self.eof:
+ self.fail("Could not find end of expression starting at %d."
+ % startPos)
+ value = self.definition[startPos:self.pos].strip()
+ return ASTFallbackExpr(value.strip())
+
+ def _parse_nested_name(self) -> ASTNestedName:
+ names: list[Any] = []
+
+ self.skip_ws()
+ rooted = False
+ if self.skip_string('.'):
+ rooted = True
+ while 1:
+ self.skip_ws()
+ if not self.match(identifier_re):
+ self.fail("Expected identifier in nested name.")
+ identifier = self.matched_text
+ # make sure there isn't a keyword
+ if identifier in _keywords:
+ self.fail("Expected identifier in nested name, "
+ "got keyword: %s" % identifier)
+ if self.matched_text in self.config.c_extra_keywords:
+ msg = (
+ 'Expected identifier, got user-defined keyword: %s.'
+ ' Remove it from c_extra_keywords to allow it as identifier.\n'
+ 'Currently c_extra_keywords is %s.'
+ )
+ self.fail(msg % (self.matched_text,
+ str(self.config.c_extra_keywords)))
+ ident = ASTIdentifier(identifier)
+ names.append(ident)
+
+ self.skip_ws()
+ if not self.skip_string('.'):
+ break
+ return ASTNestedName(names, rooted)
+
+ def _parse_simple_type_specifier(self) -> str | None:
+ if self.match(_simple_type_specifiers_re):
+ return self.matched_text
+ for t in ('bool', 'complex', 'imaginary'):
+ if t in self.config.c_extra_keywords:
+ if self.skip_word(t):
+ return t
+ return None
+
+ def _parse_simple_type_specifiers(self) -> ASTTrailingTypeSpecFundamental | None:
+ names: list[str] = []
+
+ self.skip_ws()
+ while True:
+ t = self._parse_simple_type_specifier()
+ if t is None:
+ break
+ names.append(t)
+ self.skip_ws()
+ if len(names) == 0:
+ return None
+ return ASTTrailingTypeSpecFundamental(names)
- def _parse_decl_specs_simple(self, outer: (str | None), typed: bool
- ) ->ASTDeclSpecsSimple:
+ def _parse_trailing_type_spec(self) -> ASTTrailingTypeSpec:
+ # fundamental types, https://en.cppreference.com/w/c/language/type
+ # and extensions
+ self.skip_ws()
+ res = self._parse_simple_type_specifiers()
+ if res is not None:
+ return res
+
+ # prefixed
+ prefix = None
+ self.skip_ws()
+ for k in ('struct', 'enum', 'union'):
+ if self.skip_word_and_ws(k):
+ prefix = k
+ break
+
+ nestedName = self._parse_nested_name()
+ return ASTTrailingTypeSpecName(prefix, nestedName)
+
+ def _parse_parameters(self, paramMode: str) -> ASTParameters | None:
+ self.skip_ws()
+ if not self.skip_string('('):
+ if paramMode == 'function':
+ self.fail('Expecting "(" in parameters.')
+ else:
+ return None
+
+ args = []
+ self.skip_ws()
+ if not self.skip_string(')'):
+ while 1:
+ self.skip_ws()
+ if self.skip_string('...'):
+ args.append(ASTFunctionParameter(None, True))
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail('Expected ")" after "..." in parameters.')
+ break
+ # note: it seems that function arguments can always be named,
+ # even in function pointers and similar.
+ arg = self._parse_type_with_init(outer=None, named='single')
+ # TODO: parse default parameters # TODO: didn't we just do that?
+ args.append(ASTFunctionParameter(arg))
+
+ self.skip_ws()
+ if self.skip_string(','):
+ continue
+ if self.skip_string(')'):
+ break
+ self.fail(f'Expecting "," or ")" in parameters, got "{self.current_char}".')
+
+ attrs = self._parse_attribute_list()
+ return ASTParameters(args, attrs)
+
+ def _parse_decl_specs_simple(
+ self, outer: str | None, typed: bool,
+ ) -> ASTDeclSpecsSimple:
"""Just parse the simple ones."""
- pass
+ storage = None
+ threadLocal = None
+ inline = None
+ restrict = None
+ volatile = None
+ const = None
+ attrs = []
+ while 1: # accept any permutation of a subset of some decl-specs
+ self.skip_ws()
+ if not storage:
+ if outer == 'member':
+ if self.skip_word('auto'):
+ storage = 'auto'
+ continue
+ if self.skip_word('register'):
+ storage = 'register'
+ continue
+ if outer in ('member', 'function'):
+ if self.skip_word('static'):
+ storage = 'static'
+ continue
+ if self.skip_word('extern'):
+ storage = 'extern'
+ continue
+ if outer == 'member' and not threadLocal:
+ if self.skip_word('thread_local'):
+ threadLocal = 'thread_local'
+ continue
+ if self.skip_word('_Thread_local'):
+ threadLocal = '_Thread_local'
+ continue
+ if outer == 'function' and not inline:
+ inline = self.skip_word('inline')
+ if inline:
+ continue
+
+ if not restrict and typed:
+ restrict = self.skip_word('restrict')
+ if restrict:
+ continue
+ if not volatile and typed:
+ volatile = self.skip_word('volatile')
+ if volatile:
+ continue
+ if not const and typed:
+ const = self.skip_word('const')
+ if const:
+ continue
+ attr = self._parse_attribute()
+ if attr:
+ attrs.append(attr)
+ continue
+ break
+ return ASTDeclSpecsSimple(storage, threadLocal, inline,
+ restrict, volatile, const, ASTAttributeList(attrs))
+
+ def _parse_decl_specs(self, outer: str | None, typed: bool = True) -> ASTDeclSpecs:
+ if outer:
+ if outer not in ('type', 'member', 'function'):
+ raise Exception('Internal error, unknown outer "%s".' % outer)
+ leftSpecs = self._parse_decl_specs_simple(outer, typed)
+ rightSpecs = None
+
+ if typed:
+ trailing = self._parse_trailing_type_spec()
+ rightSpecs = self._parse_decl_specs_simple(outer, typed)
+ else:
+ trailing = None
+ return ASTDeclSpecs(outer, leftSpecs, rightSpecs, trailing)
+
+ def _parse_declarator_name_suffix(
+ self, named: bool | str, paramMode: str, typed: bool,
+ ) -> ASTDeclarator:
+ assert named in (True, False, 'single')
+ # now we should parse the name, and then suffixes
+ if named == 'single':
+ if self.match(identifier_re):
+ if self.matched_text in _keywords:
+ self.fail("Expected identifier, "
+ "got keyword: %s" % self.matched_text)
+ if self.matched_text in self.config.c_extra_keywords:
+ msg = (
+ 'Expected identifier, got user-defined keyword: %s. '
+ 'Remove it from c_extra_keywords to allow it as identifier.\n'
+ 'Currently c_extra_keywords is %s.'
+ )
+ self.fail(msg % (self.matched_text,
+ str(self.config.c_extra_keywords)))
+ identifier = ASTIdentifier(self.matched_text)
+ declId = ASTNestedName([identifier], rooted=False)
+ else:
+ declId = None
+ elif named:
+ declId = self._parse_nested_name()
+ else:
+ declId = None
+ arrayOps = []
+ while 1:
+ self.skip_ws()
+ if typed and self.skip_string('['):
+ self.skip_ws()
+ static = False
+ const = False
+ volatile = False
+ restrict = False
+ while True:
+ if not static:
+ if self.skip_word_and_ws('static'):
+ static = True
+ continue
+ if not const:
+ if self.skip_word_and_ws('const'):
+ const = True
+ continue
+ if not volatile:
+ if self.skip_word_and_ws('volatile'):
+ volatile = True
+ continue
+ if not restrict:
+ if self.skip_word_and_ws('restrict'):
+ restrict = True
+ continue
+ break
+ vla = False if static else self.skip_string_and_ws('*')
+ if vla:
+ if not self.skip_string(']'):
+ self.fail("Expected ']' in end of array operator.")
+ size = None
+ else:
+ if self.skip_string(']'):
+ size = None
+ else:
+
+ def parser() -> ASTExpression:
+ return self._parse_expression()
+ size = self._parse_expression_fallback([']'], parser)
+ self.skip_ws()
+ if not self.skip_string(']'):
+ self.fail("Expected ']' in end of array operator.")
+ arrayOps.append(ASTArray(static, const, volatile, restrict, vla, size))
+ else:
+ break
+ param = self._parse_parameters(paramMode)
+ if param is None and len(arrayOps) == 0:
+ # perhaps a bit-field
+ if named and paramMode == 'type' and typed:
+ self.skip_ws()
+ if self.skip_string(':'):
+ size = self._parse_constant_expression()
+ return ASTDeclaratorNameBitField(declId=declId, size=size)
+ return ASTDeclaratorNameParam(declId=declId, arrayOps=arrayOps,
+ param=param)
+
+ def _parse_declarator(self, named: bool | str, paramMode: str,
+ typed: bool = True) -> ASTDeclarator:
+ # 'typed' here means 'parse return type stuff'
+ if paramMode not in ('type', 'function'):
+ raise Exception(
+ "Internal error, unknown paramMode '%s'." % paramMode)
+ prevErrors = []
+ self.skip_ws()
+ if typed and self.skip_string('*'):
+ self.skip_ws()
+ restrict = False
+ volatile = False
+ const = False
+ attrs = []
+ while 1:
+ if not restrict:
+ restrict = self.skip_word_and_ws('restrict')
+ if restrict:
+ continue
+ if not volatile:
+ volatile = self.skip_word_and_ws('volatile')
+ if volatile:
+ continue
+ if not const:
+ const = self.skip_word_and_ws('const')
+ if const:
+ continue
+ attr = self._parse_attribute()
+ if attr is not None:
+ attrs.append(attr)
+ continue
+ break
+ next = self._parse_declarator(named, paramMode, typed)
+ return ASTDeclaratorPtr(next=next,
+ restrict=restrict, volatile=volatile, const=const,
+ attrs=ASTAttributeList(attrs))
+ if typed and self.current_char == '(': # note: peeking, not skipping
+ # maybe this is the beginning of params, try that first,
+ # otherwise assume it's noptr->declarator > ( ptr-declarator )
+ pos = self.pos
+ try:
+ # assume this is params
+ res = self._parse_declarator_name_suffix(named, paramMode,
+ typed)
+ return res
+ except DefinitionError as exParamQual:
+ msg = "If declarator-id with parameters"
+ if paramMode == 'function':
+ msg += " (e.g., 'void f(int arg)')"
+ prevErrors.append((exParamQual, msg))
+ self.pos = pos
+ try:
+ assert self.current_char == '('
+ self.skip_string('(')
+ # TODO: hmm, if there is a name, it must be in inner, right?
+ # TODO: hmm, if there must be parameters, they must b
+ # inside, right?
+ inner = self._parse_declarator(named, paramMode, typed)
+ if not self.skip_string(')'):
+ self.fail("Expected ')' in \"( ptr-declarator )\"")
+ next = self._parse_declarator(named=False,
+ paramMode="type",
+ typed=typed)
+ return ASTDeclaratorParen(inner=inner, next=next)
+ except DefinitionError as exNoPtrParen:
+ self.pos = pos
+ msg = "If parenthesis in noptr-declarator"
+ if paramMode == 'function':
+ msg += " (e.g., 'void (*f(int arg))(double)')"
+ prevErrors.append((exNoPtrParen, msg))
+ header = "Error in declarator"
+ raise self._make_multi_error(prevErrors, header) from exNoPtrParen
+ pos = self.pos
+ try:
+ return self._parse_declarator_name_suffix(named, paramMode, typed)
+ except DefinitionError as e:
+ self.pos = pos
+ prevErrors.append((e, "If declarator-id"))
+ header = "Error in declarator or parameters"
+ raise self._make_multi_error(prevErrors, header) from e
+
+ def _parse_initializer(self, outer: str | None = None, allowFallback: bool = True,
+ ) -> ASTInitializer | None:
+ self.skip_ws()
+ if outer == 'member' and False: # NoQA: SIM223 # TODO
+ bracedInit = self._parse_braced_init_list()
+ if bracedInit is not None:
+ return ASTInitializer(bracedInit, hasAssign=False)
+
+ if not self.skip_string('='):
+ return None
+
+ bracedInit = self._parse_braced_init_list()
+ if bracedInit is not None:
+ return ASTInitializer(bracedInit)
+
+ if outer == 'member':
+ fallbackEnd: list[str] = []
+ elif outer is None: # function parameter
+ fallbackEnd = [',', ')']
+ else:
+ self.fail("Internal error, initializer for outer '%s' not "
+ "implemented." % outer)
- def _parse_type(self, named: (bool | str), outer: (str | None)=None
- ) ->ASTType:
+ def parser() -> ASTExpression:
+ return self._parse_assignment_expression()
+
+ value = self._parse_expression_fallback(fallbackEnd, parser, allow=allowFallback)
+ return ASTInitializer(value)
+
+ def _parse_type(self, named: bool | str, outer: str | None = None) -> ASTType:
"""
named=False|'single'|True: 'single' is e.g., for function objects which
doesn't need to name the arguments, but otherwise is a single name
"""
- pass
+ if outer: # always named
+ if outer not in ('type', 'member', 'function'):
+ raise Exception('Internal error, unknown outer "%s".' % outer)
+ assert named
+
+ if outer == 'type':
+ # We allow type objects to just be a name.
+ prevErrors = []
+ startPos = self.pos
+ # first try without the type
+ try:
+ declSpecs = self._parse_decl_specs(outer=outer, typed=False)
+ decl = self._parse_declarator(named=True, paramMode=outer,
+ typed=False)
+ self.assert_end(allowSemicolon=True)
+ except DefinitionError as exUntyped:
+ desc = "If just a name"
+ prevErrors.append((exUntyped, desc))
+ self.pos = startPos
+ try:
+ declSpecs = self._parse_decl_specs(outer=outer)
+ decl = self._parse_declarator(named=True, paramMode=outer)
+ except DefinitionError as exTyped:
+ self.pos = startPos
+ desc = "If typedef-like declaration"
+ prevErrors.append((exTyped, desc))
+ # Retain the else branch for easier debugging.
+ # TODO: it would be nice to save the previous stacktrace
+ # and output it here.
+ if True:
+ header = "Type must be either just a name or a "
+ header += "typedef-like declaration."
+ raise self._make_multi_error(prevErrors, header) from exTyped
+ else: # NoQA: RET506
+ # For testing purposes.
+ # do it again to get the proper traceback (how do you
+ # reliably save a traceback when an exception is
+ # constructed?)
+ self.pos = startPos
+ typed = True
+ declSpecs = self._parse_decl_specs(outer=outer, typed=typed)
+ decl = self._parse_declarator(named=True, paramMode=outer,
+ typed=typed)
+ elif outer == 'function':
+ declSpecs = self._parse_decl_specs(outer=outer)
+ decl = self._parse_declarator(named=True, paramMode=outer)
+ else:
+ paramMode = 'type'
+ if outer == 'member': # i.e., member
+ named = True
+ declSpecs = self._parse_decl_specs(outer=outer)
+ decl = self._parse_declarator(named=named, paramMode=paramMode)
+ return ASTType(declSpecs, decl)
+
+ def _parse_type_with_init(self, named: bool | str, outer: str | None) -> ASTTypeWithInit:
+ if outer:
+ assert outer in ('type', 'member', 'function')
+ type = self._parse_type(outer=outer, named=named)
+ init = self._parse_initializer(outer=outer)
+ return ASTTypeWithInit(type, init)
+
+ def _parse_macro(self) -> ASTMacro:
+ self.skip_ws()
+ ident = self._parse_nested_name()
+ if ident is None:
+ self.fail("Expected identifier in macro definition.")
+ self.skip_ws()
+ if not self.skip_string_and_ws('('):
+ return ASTMacro(ident, None)
+ if self.skip_string(')'):
+ return ASTMacro(ident, [])
+ args = []
+ while 1:
+ self.skip_ws()
+ if self.skip_string('...'):
+ args.append(ASTMacroParameter(None, True))
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail('Expected ")" after "..." in macro parameters.')
+ break
+ if not self.match(identifier_re):
+ self.fail("Expected identifier in macro parameters.")
+ nn = ASTNestedName([ASTIdentifier(self.matched_text)], rooted=False)
+ # Allow named variadic args:
+ # https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html
+ self.skip_ws()
+ if self.skip_string_and_ws('...'):
+ args.append(ASTMacroParameter(nn, False, True))
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail('Expected ")" after "..." in macro parameters.')
+ break
+ args.append(ASTMacroParameter(nn))
+ if self.skip_string_and_ws(','):
+ continue
+ if self.skip_string_and_ws(')'):
+ break
+ self.fail("Expected identifier, ')', or ',' in macro parameter list.")
+ return ASTMacro(ident, args)
+
+ def _parse_struct(self) -> ASTStruct:
+ name = self._parse_nested_name()
+ return ASTStruct(name)
+
+ def _parse_union(self) -> ASTUnion:
+ name = self._parse_nested_name()
+ return ASTUnion(name)
+
+ def _parse_enum(self) -> ASTEnum:
+ name = self._parse_nested_name()
+ return ASTEnum(name)
+
+ def _parse_enumerator(self) -> ASTEnumerator:
+ name = self._parse_nested_name()
+ attrs = self._parse_attribute_list()
+ self.skip_ws()
+ init = None
+ if self.skip_string('='):
+ self.skip_ws()
+
+ def parser() -> ASTExpression:
+ return self._parse_constant_expression()
+
+ initVal = self._parse_expression_fallback([], parser)
+ init = ASTInitializer(initVal)
+ return ASTEnumerator(name, init, attrs)
+
+ def parse_declaration(self, objectType: str, directiveType: str) -> ASTDeclaration:
+ if objectType not in ('function', 'member',
+ 'macro', 'struct', 'union', 'enum', 'enumerator', 'type'):
+ raise Exception('Internal error, unknown objectType "%s".' % objectType)
+ if directiveType not in ('function', 'member', 'var',
+ 'macro', 'struct', 'union', 'enum', 'enumerator', 'type'):
+ raise Exception('Internal error, unknown directiveType "%s".' % directiveType)
+
+ declaration: DeclarationType | None = None
+ if objectType == 'member':
+ declaration = self._parse_type_with_init(named=True, outer='member')
+ elif objectType == 'function':
+ declaration = self._parse_type(named=True, outer='function')
+ elif objectType == 'macro':
+ declaration = self._parse_macro()
+ elif objectType == 'struct':
+ declaration = self._parse_struct()
+ elif objectType == 'union':
+ declaration = self._parse_union()
+ elif objectType == 'enum':
+ declaration = self._parse_enum()
+ elif objectType == 'enumerator':
+ declaration = self._parse_enumerator()
+ elif objectType == 'type':
+ declaration = self._parse_type(named=True, outer='type')
+ else:
+ raise AssertionError
+ if objectType != 'macro':
+ self.skip_ws()
+ semicolon = self.skip_string(';')
+ else:
+ semicolon = False
+ return ASTDeclaration(objectType, directiveType, declaration, semicolon)
+
+ def parse_namespace_object(self) -> ASTNestedName:
+ return self._parse_nested_name()
+
+ def parse_xref_object(self) -> ASTNestedName:
+ name = self._parse_nested_name()
+ # if there are '()' left, just skip them
+ self.skip_ws()
+ self.skip_string('()')
+ self.assert_end()
+ return name
+
+ def parse_expression(self) -> ASTExpression | ASTType:
+ pos = self.pos
+ res: ASTExpression | ASTType | None = None
+ try:
+ res = self._parse_expression()
+ self.skip_ws()
+ self.assert_end()
+ except DefinitionError as exExpr:
+ self.pos = pos
+ try:
+ res = self._parse_type(False)
+ self.skip_ws()
+ self.assert_end()
+ except DefinitionError as exType:
+ header = "Error when parsing (type) expression."
+ errs = []
+ errs.append((exExpr, "If expression"))
+ errs.append((exType, "If type"))
+ raise self._make_multi_error(errs, header) from exType
+ return res
diff --git a/sphinx/domains/c/_symbol.py b/sphinx/domains/c/_symbol.py
index 8ce95af65..c70b51316 100644
--- a/sphinx/domains/c/_symbol.py
+++ b/sphinx/domains/c/_symbol.py
@@ -1,69 +1,98 @@
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any
-from sphinx.domains.c._ast import ASTDeclaration, ASTIdentifier, ASTNestedName
+
+from sphinx.domains.c._ast import (
+ ASTDeclaration,
+ ASTIdentifier,
+ ASTNestedName,
+)
from sphinx.locale import __
from sphinx.util import logging
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Iterator, Sequence
+
from typing_extensions import Self
+
from sphinx.environment import BuildEnvironment
+
logger = logging.getLogger(__name__)
class _DuplicateSymbolError(Exception):
-
- def __init__(self, symbol: Symbol, declaration: ASTDeclaration) ->None:
+ def __init__(self, symbol: Symbol, declaration: ASTDeclaration) -> None:
assert symbol
assert declaration
self.symbol = symbol
self.declaration = declaration
- def __str__(self) ->str:
- return 'Internal C duplicate symbol error:\n%s' % self.symbol.dump(0)
+ def __str__(self) -> str:
+ return "Internal C duplicate symbol error:\n%s" % self.symbol.dump(0)
class SymbolLookupResult:
-
def __init__(self, symbols: Sequence[Symbol], parentSymbol: Symbol,
- ident: ASTIdentifier) ->None:
+ ident: ASTIdentifier) -> None:
self.symbols = symbols
self.parentSymbol = parentSymbol
self.ident = ident
class LookupKey:
-
- def __init__(self, data: list[tuple[ASTIdentifier, str]]) ->None:
+ def __init__(self, data: list[tuple[ASTIdentifier, str]]) -> None:
self.data = data
- def __str__(self) ->str:
- inner = ', '.join(f'({ident}, {id_})' for ident, id_ in self.data)
+ def __str__(self) -> str:
+ inner = ', '.join(f"({ident}, {id_})" for ident, id_ in self.data)
return f'[{inner}]'
class Symbol:
debug_indent = 0
- debug_indent_string = ' '
+ debug_indent_string = " "
debug_lookup = False
debug_show_tree = False
- def __copy__(self) ->Self:
- raise AssertionError
+ def __copy__(self) -> Self:
+ raise AssertionError # shouldn't happen
- def __deepcopy__(self, memo: Any) ->Symbol:
+ def __deepcopy__(self, memo: Any) -> Symbol:
if self.parent:
- raise AssertionError
+ raise AssertionError # shouldn't happen
+ # the domain base class makes a copy of the initial data, which is fine
return Symbol(None, None, None, None, None)
- def __setattr__(self, key: str, value: Any) ->None:
- if key == 'children':
+ @staticmethod
+ def debug_print(*args: Any) -> None:
+ msg = Symbol.debug_indent_string * Symbol.debug_indent
+ msg += "".join(str(e) for e in args)
+ logger.debug(msg)
+
+ def _assert_invariants(self) -> None:
+ if not self.parent:
+ # parent == None means global scope, so declaration means a parent
+ assert not self.declaration
+ assert not self.docname
+ else:
+ if self.declaration:
+ assert self.docname
+
+ def __setattr__(self, key: str, value: Any) -> None:
+ if key == "children":
raise AssertionError
return super().__setattr__(key, value)
- def __init__(self, parent: (Symbol | None), ident: (ASTIdentifier |
- None), declaration: (ASTDeclaration | None), docname: (str | None),
- line: (int | None)) ->None:
+ def __init__(
+ self,
+ parent: Symbol | None,
+ ident: ASTIdentifier | None,
+ declaration: ASTDeclaration | None,
+ docname: str | None,
+ line: int | None,
+ ) -> None:
self.parent = parent
+ # declarations in a single directive are linked together
self.siblingAbove: Symbol | None = None
self.siblingBelow: Symbol | None = None
self.ident = ident
@@ -72,14 +101,555 @@ class Symbol:
self.line = line
self.isRedeclaration = False
self._assert_invariants()
+
+ # These properties store the same children for different access patterns.
+ # ``_add_child()`` and ``_remove_child()`` should be used for modifying them.
self._children_by_name: dict[str, Symbol] = {}
self._children_by_docname: dict[str, dict[str, Symbol]] = {}
self._anon_children: set[Symbol] = set()
+
if self.parent:
self.parent._add_child(self)
if self.declaration:
self.declaration.symbol = self
+
+ # Do symbol addition after self._children has been initialised.
self._add_function_params()
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return f'<Symbol {self.to_string(indent=0)!r}>'
+
+ @property
+ def _children(self) -> Iterable[Symbol]:
+ return self._children_by_name.values()
+
+ def _add_child(self, child: Symbol) -> None:
+ name = child.ident.name
+ if name in self._children_by_name:
+ # Duplicate so don't add - will be reported in _add_symbols()
+ return
+ self._children_by_name[name] = child
+ self._children_by_docname.setdefault(child.docname, {})[name] = child
+ if child.ident.is_anonymous:
+ self._anon_children.add(child)
+
+ def _remove_child(self, child: Symbol) -> None:
+ name = child.ident.name
+ self._children_by_name.pop(name, None)
+ self._children_by_docname.get(child.docname, {}).pop(name, None)
+ if child.ident.is_anonymous:
+ self._anon_children.discard(child)
+
+ def _fill_empty(self, declaration: ASTDeclaration, docname: str, line: int) -> None:
+ self._assert_invariants()
+ assert self.declaration is None
+ assert self.docname is None
+ assert self.line is None
+ assert declaration is not None
+ assert docname is not None
+ assert line is not None
+ self.declaration = declaration
+ self.declaration.symbol = self
+ self.docname = docname
+ self.line = line
+ self._assert_invariants()
+ # and symbol addition should be done as well
+ self._add_function_params()
+
+ def _add_function_params(self) -> None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_add_function_params:")
+ # Note: we may be called from _fill_empty, so the symbols we want
+ # to add may actually already be present (as empty symbols).
+
+ # add symbols for function parameters, if any
+ if self.declaration is not None and self.declaration.function_params is not None:
+ for p in self.declaration.function_params:
+ if p.arg is None:
+ continue
+ nn = p.arg.name
+ if nn is None:
+ continue
+ # (comparing to the template params: we have checked that we are a declaration)
+ decl = ASTDeclaration('functionParam', None, p)
+ assert not nn.rooted
+ assert len(nn.names) == 1
+ self._add_symbols(nn, decl, self.docname, self.line)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+
+ def remove(self) -> None:
+ if self.parent:
+ self.parent._remove_child(self)
+ self.parent = None
+
+ def clear_doc(self, docname: str) -> None:
+ if docname not in self._children_by_docname:
+ for child in self._children:
+ child.clear_doc(docname)
+ return
+
+ children: dict[str, Symbol] = self._children_by_docname.pop(docname)
+ for child in children.values():
+ child.declaration = None
+ child.docname = None
+ child.line = None
+ if child.siblingAbove is not None:
+ child.siblingAbove.siblingBelow = child.siblingBelow
+ if child.siblingBelow is not None:
+ child.siblingBelow.siblingAbove = child.siblingAbove
+ child.siblingAbove = None
+ child.siblingBelow = None
+ self._remove_child(child)
+
+ def get_all_symbols(self) -> Iterator[Symbol]:
+ yield self
+ for sChild in self._children:
+ yield from sChild.get_all_symbols()
+
+ @property
+ def children(self) -> Iterator[Symbol]:
+ yield from self._children
+
+ def get_lookup_key(self) -> LookupKey:
+ # The pickle files for the environment and for each document are distinct.
+ # The environment has all the symbols, but the documents has xrefs that
+ # must know their scope. A lookup key is essentially a specification of
+ # how to find a specific symbol.
+ symbols = []
+ s = self
+ while s.parent:
+ symbols.append(s)
+ s = s.parent
+ symbols.reverse()
+ key = []
+ for s in symbols:
+ if s.declaration is not None:
+ # TODO: do we need the ID?
+ key.append((s.ident, s.declaration.get_newest_id()))
+ else:
+ key.append((s.ident, None))
+ return LookupKey(key)
+
+ def get_full_nested_name(self) -> ASTNestedName:
+ symbols = []
+ s = self
+ while s.parent:
+ symbols.append(s)
+ s = s.parent
+ symbols.reverse()
+ names = [s.ident for s in symbols]
+ return ASTNestedName(names, rooted=False)
+
+ def _symbol_lookup(
+ self,
+ nestedName: ASTNestedName,
+ onMissingQualifiedSymbol: Callable[[Symbol, ASTIdentifier], Symbol | None],
+ ancestorLookupType: str | None,
+ matchSelf: bool,
+ recurseInAnon: bool,
+ searchInSiblings: bool,
+ ) -> SymbolLookupResult | None:
+ # TODO: further simplification from C++ to C
+ # ancestorLookupType: if not None, specifies the target type of the lookup
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_symbol_lookup:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("self:")
+ logger.debug(self.to_string(Symbol.debug_indent + 1, addEndNewline=False))
+ Symbol.debug_print("nestedName: ", nestedName)
+ Symbol.debug_print("ancestorLookupType:", ancestorLookupType)
+ Symbol.debug_print("matchSelf: ", matchSelf)
+ Symbol.debug_print("recurseInAnon: ", recurseInAnon)
+ Symbol.debug_print("searchInSiblings: ", searchInSiblings)
+
+ names = nestedName.names
+
+ # find the right starting point for lookup
+ parentSymbol = self
+ if nestedName.rooted:
+ while parentSymbol.parent is not None:
+ parentSymbol = parentSymbol.parent
+
+ if ancestorLookupType is not None:
+ # walk up until we find the first identifier
+ firstName = names[0]
+ while parentSymbol.parent:
+ if firstName.name in parentSymbol._children_by_name:
+ break
+ parentSymbol = parentSymbol.parent
+
+ if Symbol.debug_lookup:
+ Symbol.debug_print("starting point:")
+ logger.debug(parentSymbol.to_string(Symbol.debug_indent + 1, addEndNewline=False))
+
+ # and now the actual lookup
+ for ident in names[:-1]:
+ name = ident.name
+ if name in parentSymbol._children_by_name:
+ symbol = parentSymbol._children_by_name[name]
+ else:
+ symbol = onMissingQualifiedSymbol(parentSymbol, ident)
+ if symbol is None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return None
+ parentSymbol = symbol
+
+ if Symbol.debug_lookup:
+ Symbol.debug_print("handle last name from:")
+ logger.debug(parentSymbol.to_string(Symbol.debug_indent + 1, addEndNewline=False))
+
+ # handle the last name
+ ident = names[-1]
+ name = ident.name
+ symbol = parentSymbol._children_by_name.get(name)
+ if not symbol and recurseInAnon:
+ for child in parentSymbol._anon_children:
+ if name in child._children_by_name:
+ symbol = child._children_by_name[name]
+ break
+
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+
+ result = [symbol] if symbol else []
+ return SymbolLookupResult(result, parentSymbol, ident)
+
+ def _add_symbols(
+ self,
+ nestedName: ASTNestedName,
+ declaration: ASTDeclaration | None,
+ docname: str | None,
+ line: int | None,
+ ) -> Symbol:
+ # TODO: further simplification from C++ to C
+ # Used for adding a whole path of symbols, where the last may or may not
+ # be an actual declaration.
+
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_add_symbols:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("nn: ", nestedName)
+ Symbol.debug_print("decl: ", declaration)
+ Symbol.debug_print(f"location: {docname}:{line}")
+
+ def onMissingQualifiedSymbol(parentSymbol: Symbol, ident: ASTIdentifier) -> Symbol:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_add_symbols, onMissingQualifiedSymbol:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("ident: ", ident)
+ Symbol.debug_indent -= 2
+ return Symbol(parent=parentSymbol, ident=ident,
+ declaration=None, docname=None, line=None)
+
+ lookupResult = self._symbol_lookup(nestedName,
+ onMissingQualifiedSymbol,
+ ancestorLookupType=None,
+ matchSelf=False,
+ recurseInAnon=False,
+ searchInSiblings=False)
+ assert lookupResult is not None # we create symbols all the way, so that can't happen
+ symbols = list(lookupResult.symbols)
+ if len(symbols) == 0:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("_add_symbols, result, no symbol:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("ident: ", lookupResult.ident)
+ Symbol.debug_print("declaration: ", declaration)
+ Symbol.debug_print(f"location: {docname}:{line}")
+ Symbol.debug_indent -= 1
+ symbol = Symbol(parent=lookupResult.parentSymbol,
+ ident=lookupResult.ident,
+ declaration=declaration,
+ docname=docname, line=line)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return symbol
+
+ if Symbol.debug_lookup:
+ Symbol.debug_print("_add_symbols, result, symbols:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("number symbols:", len(symbols))
+ Symbol.debug_indent -= 1
+
+ if not declaration:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("no declaration")
+ Symbol.debug_indent -= 2
+ # good, just a scope creation
+ # TODO: what if we have more than one symbol?
+ return symbols[0]
+
+ noDecl = []
+ withDecl = []
+ dupDecl = []
+ for s in symbols:
+ if s.declaration is None:
+ noDecl.append(s)
+ elif s.isRedeclaration:
+ dupDecl.append(s)
+ else:
+ withDecl.append(s)
+ if Symbol.debug_lookup:
+ Symbol.debug_print("#noDecl: ", len(noDecl))
+ Symbol.debug_print("#withDecl:", len(withDecl))
+ Symbol.debug_print("#dupDecl: ", len(dupDecl))
+
+ # With partial builds we may start with a large symbol tree stripped of declarations.
+ # Essentially any combination of noDecl, withDecl, and dupDecls seems possible.
+ # TODO: make partial builds fully work. What should happen when the primary symbol gets
+ # deleted, and other duplicates exist? The full document should probably be rebuild.
+
+ # First check if one of those with a declaration matches.
+ # If it's a function, we need to compare IDs,
+ # otherwise there should be only one symbol with a declaration.
+ def makeCandSymbol() -> Symbol:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("begin: creating candidate symbol")
+ symbol = Symbol(parent=lookupResult.parentSymbol,
+ ident=lookupResult.ident,
+ declaration=declaration,
+ docname=docname, line=line)
+ if Symbol.debug_lookup:
+ Symbol.debug_print("end: creating candidate symbol")
+ return symbol
+
+ if len(withDecl) == 0:
+ candSymbol = None
+ else:
+ candSymbol = makeCandSymbol()
+
+ def handleDuplicateDeclaration(symbol: Symbol, candSymbol: Symbol) -> None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("redeclaration")
+ Symbol.debug_indent -= 1
+ Symbol.debug_indent -= 2
+ # Redeclaration of the same symbol.
+ # Let the new one be there, but raise an error to the client
+ # so it can use the real symbol as subscope.
+ # This will probably result in a duplicate id warning.
+ candSymbol.isRedeclaration = True
+ raise _DuplicateSymbolError(symbol, declaration)
+
+ if declaration.objectType != "function":
+ assert len(withDecl) <= 1
+ handleDuplicateDeclaration(withDecl[0], candSymbol)
+ # (not reachable)
+
+ # a function, so compare IDs
+ candId = declaration.get_newest_id()
+ if Symbol.debug_lookup:
+ Symbol.debug_print("candId:", candId)
+ for symbol in withDecl:
+ oldId = symbol.declaration.get_newest_id()
+ if Symbol.debug_lookup:
+ Symbol.debug_print("oldId: ", oldId)
+ if candId == oldId:
+ handleDuplicateDeclaration(symbol, candSymbol)
+ # (not reachable)
+ # no candidate symbol found with matching ID
+ # if there is an empty symbol, fill that one
+ if len(noDecl) == 0:
+ if Symbol.debug_lookup:
+ Symbol.debug_print(
+ "no match, no empty, candSybmol is not None?:", candSymbol is not None,
+ )
+ Symbol.debug_indent -= 2
+ if candSymbol is not None:
+ return candSymbol
+ else:
+ return makeCandSymbol()
+ else:
+ if Symbol.debug_lookup:
+ Symbol.debug_print(
+ "no match, but fill an empty declaration, candSybmol is not None?:",
+ candSymbol is not None)
+ Symbol.debug_indent -= 2
+ if candSymbol is not None:
+ candSymbol.remove()
+ # assert len(noDecl) == 1
+ # TODO: enable assertion when we at some point find out how to do cleanup
+ # for now, just take the first one, it should work fine ... right?
+ symbol = noDecl[0]
+ # If someone first opened the scope, and then later
+ # declares it, e.g,
+ # .. namespace:: Test
+ # .. namespace:: nullptr
+ # .. class:: Test
+ symbol._fill_empty(declaration, docname, line)
+ return symbol
+
+ def merge_with(self, other: Symbol, docnames: list[str],
+ env: BuildEnvironment) -> None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("merge_with:")
+
+ assert other is not None
+ for otherChild in other._children:
+ otherName = otherChild.ident.name
+ if otherName not in self._children_by_name:
+ # TODO: hmm, should we prune by docnames?
+ otherChild.parent = self
+ self._add_child(otherChild)
+ otherChild._assert_invariants()
+ continue
+ ourChild = self._children_by_name[otherName]
+ if otherChild.declaration and otherChild.docname in docnames:
+ if not ourChild.declaration:
+ ourChild._fill_empty(otherChild.declaration,
+ otherChild.docname, otherChild.line)
+ elif ourChild.docname != otherChild.docname:
+ name = str(ourChild.declaration)
+ msg = __("Duplicate C declaration, also defined at %s:%s.\n"
+ "Declaration is '.. c:%s:: %s'.")
+ msg = msg % (ourChild.docname, ourChild.line,
+ ourChild.declaration.directiveType, name)
+ logger.warning(msg, location=(otherChild.docname, otherChild.line))
+ else:
+ # Both have declarations, and in the same docname.
+ # This can apparently happen, it should be safe to
+ # just ignore it, right?
+ pass
+ ourChild.merge_with(otherChild, docnames, env)
+
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+
+ def add_name(self, nestedName: ASTNestedName) -> Symbol:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("add_name:")
+ res = self._add_symbols(nestedName, declaration=None, docname=None, line=None)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+ return res
+
+ def add_declaration(self, declaration: ASTDeclaration,
+ docname: str, line: int) -> Symbol:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("add_declaration:")
+ assert declaration is not None
+ assert docname is not None
+ assert line is not None
+ nestedName = declaration.name
+ res = self._add_symbols(nestedName, declaration, docname, line)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+ return res
+
+ def find_identifier(self, ident: ASTIdentifier,
+ matchSelf: bool, recurseInAnon: bool, searchInSiblings: bool,
+ ) -> Symbol | None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("find_identifier:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("ident: ", ident)
+ Symbol.debug_print("matchSelf: ", matchSelf)
+ Symbol.debug_print("recurseInAnon: ", recurseInAnon)
+ Symbol.debug_print("searchInSiblings:", searchInSiblings)
+ logger.debug(self.to_string(Symbol.debug_indent + 1, addEndNewline=False))
+ Symbol.debug_indent -= 2
+ current = self
+ while current is not None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 2
+ Symbol.debug_print("trying:")
+ logger.debug(current.to_string(Symbol.debug_indent + 1, addEndNewline=False))
+ Symbol.debug_indent -= 2
+ if matchSelf and current.ident == ident:
+ return current
+ name = ident.name
+ if name in current._children_by_name:
+ return current._children_by_name[name]
+ if recurseInAnon:
+ for child in current._anon_children:
+ if name in child._children_by_name:
+ return child._children_by_name[name]
+ if not searchInSiblings:
+ break
+ current = current.siblingAbove
+ return None
+
+ def direct_lookup(self, key: LookupKey) -> Symbol | None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("direct_lookup:")
+ Symbol.debug_indent += 1
+ s = self
+ for ident, id_ in key.data:
+ s = s._children_by_name.get(ident.name)
+ if Symbol.debug_lookup:
+ Symbol.debug_print("name: ", ident.name)
+ Symbol.debug_print("id: ", id_)
+ if s is not None:
+ logger.debug(s.to_string(Symbol.debug_indent + 1, addEndNewline=False))
+ else:
+ Symbol.debug_print("not found")
+ if s is None:
+ break
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return s
+
+ def find_declaration(self, nestedName: ASTNestedName, typ: str,
+ matchSelf: bool, recurseInAnon: bool) -> Symbol | None:
+ # templateShorthand: missing template parameter lists for templates is ok
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("find_declaration:")
+
+ def onMissingQualifiedSymbol(
+ parentSymbol: Symbol,
+ ident: ASTIdentifier,
+ ) -> Symbol | None:
+ return None
+
+ lookupResult = self._symbol_lookup(nestedName,
+ onMissingQualifiedSymbol,
+ ancestorLookupType=typ,
+ matchSelf=matchSelf,
+ recurseInAnon=recurseInAnon,
+ searchInSiblings=False)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+ if lookupResult is None:
+ return None
+
+ symbols = list(lookupResult.symbols)
+ if len(symbols) == 0:
+ return None
+ return symbols[0]
+
+ def to_string(self, indent: int, *, addEndNewline: bool = True) -> str:
+ res = [Symbol.debug_indent_string * indent]
+ if not self.parent:
+ res.append('::')
+ else:
+ if self.ident:
+ res.append(self.ident.name)
+ else:
+ res.append(str(self.declaration))
+ if self.declaration:
+ res.append(": ")
+ if self.isRedeclaration:
+ res.append('!!duplicate!! ')
+ res.append(str(self.declaration))
+ if self.docname:
+ res.append('\t(')
+ res.append(self.docname)
+ res.append(')')
+ if addEndNewline:
+ res.append('\n')
+ return ''.join(res)
+
+ def dump(self, indent: int) -> str:
+ return ''.join([self.to_string(indent), *(c.dump(indent + 1) for c in self._children)])
diff --git a/sphinx/domains/changeset.py b/sphinx/domains/changeset.py
index dab2cbaf4..cc1d4a338 100644
--- a/sphinx/domains/changeset.py
+++ b/sphinx/domains/changeset.py
@@ -1,22 +1,37 @@
"""The changeset domain."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, cast
+
from docutils import nodes
+
from sphinx import addnodes
from sphinx.domains import Domain
from sphinx.locale import _
from sphinx.util.docutils import SphinxDirective
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata, OptionSpec
-versionlabels = {'versionadded': _('Added in version %s'), 'versionchanged':
- _('Changed in version %s'), 'deprecated': _(
- 'Deprecated since version %s'), 'versionremoved': _(
- 'Removed in version %s')}
-versionlabel_classes = {'versionadded': 'added', 'versionchanged':
- 'changed', 'deprecated': 'deprecated', 'versionremoved': 'removed'}
+
+
+versionlabels = {
+ 'versionadded': _('Added in version %s'),
+ 'versionchanged': _('Changed in version %s'),
+ 'deprecated': _('Deprecated since version %s'),
+ 'versionremoved': _('Removed in version %s'),
+}
+
+versionlabel_classes = {
+ 'versionadded': 'added',
+ 'versionchanged': 'changed',
+ 'deprecated': 'deprecated',
+ 'versionremoved': 'removed',
+}
class ChangeSet(NamedTuple):
@@ -32,15 +47,118 @@ class VersionChange(SphinxDirective):
"""
Directive to describe a change/addition/deprecation in a specific version.
"""
+
has_content = True
required_arguments = 1
optional_arguments = 1
final_argument_whitespace = True
option_spec: ClassVar[OptionSpec] = {}
+ def run(self) -> list[Node]:
+ node = addnodes.versionmodified()
+ node.document = self.state.document
+ self.set_source_info(node)
+ node['type'] = self.name
+ node['version'] = self.arguments[0]
+ text = versionlabels[self.name] % self.arguments[0]
+ if len(self.arguments) == 2:
+ inodes, messages = self.parse_inline(self.arguments[1], lineno=self.lineno + 1)
+ para = nodes.paragraph(self.arguments[1], '', *inodes, translatable=False)
+ self.set_source_info(para)
+ node.append(para)
+ else:
+ messages = []
+ if self.content:
+ node += self.parse_content_to_nodes()
+ classes = ['versionmodified', versionlabel_classes[self.name]]
+ if len(node) > 0 and isinstance(node[0], nodes.paragraph):
+ # the contents start with a paragraph
+ if node[0].rawsource:
+ # make the first paragraph translatable
+ content = nodes.inline(node[0].rawsource, translatable=True)
+ content.source = node[0].source
+ content.line = node[0].line
+ content += node[0].children
+ node[0].replace_self(nodes.paragraph('', '', content, translatable=False))
+
+ para = node[0]
+ para.insert(0, nodes.inline('', '%s: ' % text, classes=classes))
+ elif len(node) > 0:
+ # the contents do not starts with a paragraph
+ para = nodes.paragraph('', '',
+ nodes.inline('', '%s: ' % text, classes=classes),
+ translatable=False)
+ node.insert(0, para)
+ else:
+ # the contents are empty
+ para = nodes.paragraph('', '',
+ nodes.inline('', '%s.' % text, classes=classes),
+ translatable=False)
+ node.append(para)
+
+ domain = cast(ChangeSetDomain, self.env.get_domain('changeset'))
+ domain.note_changeset(node)
+
+ ret: list[Node] = [node]
+ ret += messages
+ return ret
+
class ChangeSetDomain(Domain):
"""Domain for changesets."""
+
name = 'changeset'
label = 'changeset'
- initial_data: dict[str, dict[str, list[ChangeSet]]] = {'changes': {}}
+
+ initial_data: dict[str, dict[str, list[ChangeSet]]] = {
+ 'changes': {}, # version -> list of ChangeSet
+ }
+
+ @property
+ def changesets(self) -> dict[str, list[ChangeSet]]:
+ return self.data.setdefault('changes', {}) # version -> list of ChangeSet
+
+ def note_changeset(self, node: addnodes.versionmodified) -> None:
+ version = node['version']
+ module = self.env.ref_context.get('py:module')
+ objname = self.env.temp_data.get('object')
+ changeset = ChangeSet(node['type'], self.env.docname, node.line, # type: ignore[arg-type]
+ module, objname, node.astext())
+ self.changesets.setdefault(version, []).append(changeset)
+
+ def clear_doc(self, docname: str) -> None:
+ for changes in self.changesets.values():
+ for changeset in changes.copy():
+ if changeset.docname == docname:
+ changes.remove(changeset)
+
+ def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> None:
+ # XXX duplicates?
+ for version, otherchanges in otherdata['changes'].items():
+ changes = self.changesets.setdefault(version, [])
+ for changeset in otherchanges:
+ if changeset.docname in docnames:
+ changes.append(changeset)
+
+ def process_doc(
+ self, env: BuildEnvironment, docname: str, document: nodes.document,
+ ) -> None:
+ pass # nothing to do here. All changesets are registered on calling directive.
+
+ def get_changesets_for(self, version: str) -> list[ChangeSet]:
+ return self.changesets.get(version, [])
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_domain(ChangeSetDomain)
+ app.add_directive('deprecated', VersionChange)
+ app.add_directive('versionadded', VersionChange)
+ app.add_directive('versionchanged', VersionChange)
+ app.add_directive('versionremoved', VersionChange)
+
+ return {
+ 'version': 'builtin',
+ 'env_version': 1,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/domains/citation.py b/sphinx/domains/citation.py
index f8da22b79..4f00feb81 100644
--- a/sphinx/domains/citation.py
+++ b/sphinx/domains/citation.py
@@ -1,37 +1,157 @@
"""The citation domain."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes
+
from sphinx.addnodes import pending_xref
from sphinx.domains import Domain
from sphinx.locale import __
from sphinx.transforms import SphinxTransform
from sphinx.util import logging
from sphinx.util.nodes import copy_source_info, make_refnode
+
if TYPE_CHECKING:
from docutils.nodes import Element
+
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata
+
+
logger = logging.getLogger(__name__)
class CitationDomain(Domain):
"""Domain for citations."""
+
name = 'citation'
label = 'citation'
- dangling_warnings = {'ref': 'citation not found: %(target)s'}
+
+ dangling_warnings = {
+ 'ref': 'citation not found: %(target)s',
+ }
+
+ @property
+ def citations(self) -> dict[str, tuple[str, str, int]]:
+ return self.data.setdefault('citations', {})
+
+ @property
+ def citation_refs(self) -> dict[str, set[str]]:
+ return self.data.setdefault('citation_refs', {})
+
+ def clear_doc(self, docname: str) -> None:
+ for key, (fn, _l, _lineno) in list(self.citations.items()):
+ if fn == docname:
+ del self.citations[key]
+ for key, docnames in list(self.citation_refs.items()):
+ if docnames == {docname}:
+ del self.citation_refs[key]
+ elif docname in docnames:
+ docnames.remove(docname)
+
+ def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> None:
+ # XXX duplicates?
+ for key, data in otherdata['citations'].items():
+ if data[0] in docnames:
+ self.citations[key] = data
+ for key, data in otherdata['citation_refs'].items():
+ citation_refs = self.citation_refs.setdefault(key, set())
+ for docname in data:
+ if docname in docnames:
+ citation_refs.add(docname)
+
+ def note_citation(self, node: nodes.citation) -> None:
+ label = node[0].astext()
+ if label in self.citations:
+ path = self.env.doc2path(self.citations[label][0])
+ logger.warning(__('duplicate citation %s, other instance in %s'), label, path,
+ location=node, type='ref', subtype='citation')
+ self.citations[label] = (node['docname'], node['ids'][0], node.line) # type: ignore[assignment]
+
+ def note_citation_reference(self, node: pending_xref) -> None:
+ docnames = self.citation_refs.setdefault(node['reftarget'], set())
+ docnames.add(self.env.docname)
+
+ def check_consistency(self) -> None:
+ for name, (docname, _labelid, lineno) in self.citations.items():
+ if name not in self.citation_refs:
+ logger.warning(__('Citation [%s] is not referenced.'), name,
+ type='ref', subtype='citation', location=(docname, lineno))
+
+ def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
+ typ: str, target: str, node: pending_xref, contnode: Element,
+ ) -> Element | None:
+ docname, labelid, lineno = self.citations.get(target, ('', '', 0))
+ if not docname:
+ return None
+
+ return make_refnode(builder, fromdocname, docname,
+ labelid, contnode)
+
+ def resolve_any_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
+ target: str, node: pending_xref, contnode: Element,
+ ) -> list[tuple[str, Element]]:
+ refnode = self.resolve_xref(env, fromdocname, builder, 'ref', target, node, contnode)
+ if refnode is None:
+ return []
+ else:
+ return [('ref', refnode)]
class CitationDefinitionTransform(SphinxTransform):
"""Mark citation definition labels as not smartquoted."""
+
default_priority = 619
+ def apply(self, **kwargs: Any) -> None:
+ domain = cast(CitationDomain, self.env.get_domain('citation'))
+ for node in self.document.findall(nodes.citation):
+ # register citation node to domain
+ node['docname'] = self.env.docname
+ domain.note_citation(node)
+
+ # mark citation labels as not smartquoted
+ label = cast(nodes.label, node[0])
+ label['support_smartquotes'] = False
+
class CitationReferenceTransform(SphinxTransform):
"""
Replace citation references by pending_xref nodes before the default
docutils transform tries to resolve them.
"""
+
default_priority = 619
+
+ def apply(self, **kwargs: Any) -> None:
+ domain = cast(CitationDomain, self.env.get_domain('citation'))
+ for node in self.document.findall(nodes.citation_reference):
+ target = node.astext()
+ ref = pending_xref(target, refdomain='citation', reftype='ref',
+ reftarget=target, refwarn=True,
+ support_smartquotes=False,
+ ids=node["ids"],
+ classes=node.get('classes', []))
+ ref += nodes.inline(target, '[%s]' % target)
+ copy_source_info(node, ref)
+ node.replace_self(ref)
+
+ # register reference node to domain
+ domain.note_citation_reference(ref)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_domain(CitationDomain)
+ app.add_transform(CitationDefinitionTransform)
+ app.add_transform(CitationReferenceTransform)
+
+ return {
+ 'version': 'builtin',
+ 'env_version': 1,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/domains/cpp/_ast.py b/sphinx/domains/cpp/_ast.py
index 48f498a13..e97756344 100644
--- a/sphinx/domains/cpp/_ast.py
+++ b/sphinx/domains/cpp/_ast.py
@@ -1,13 +1,36 @@
from __future__ import annotations
+
import sys
import warnings
from typing import TYPE_CHECKING, Any, ClassVar, Literal
+
from docutils import nodes
+
from sphinx import addnodes
-from sphinx.domains.cpp._ids import _id_char_from_prefix, _id_explicit_cast, _id_fundamental_v1, _id_fundamental_v2, _id_operator_unary_v2, _id_operator_v1, _id_operator_v2, _id_prefix, _id_shorthands_v1, _max_id
-from sphinx.util.cfamily import ASTAttributeList, ASTBaseBase, ASTBaseParenExprList, NoOldIdError, UnsupportedMultiCharacterCharLiteral, verify_description_mode
+from sphinx.domains.cpp._ids import (
+ _id_char_from_prefix,
+ _id_explicit_cast,
+ _id_fundamental_v1,
+ _id_fundamental_v2,
+ _id_operator_unary_v2,
+ _id_operator_v1,
+ _id_operator_v2,
+ _id_prefix,
+ _id_shorthands_v1,
+ _max_id,
+)
+from sphinx.util.cfamily import (
+ ASTAttributeList,
+ ASTBaseBase,
+ ASTBaseParenExprList,
+ NoOldIdError,
+ UnsupportedMultiCharacterCharLiteral,
+ verify_description_mode,
+)
+
if TYPE_CHECKING:
from docutils.nodes import Element, TextElement
+
from sphinx.addnodes import desc_signature
from sphinx.domains.cpp._symbol import Symbol
from sphinx.environment import BuildEnvironment
@@ -18,126 +41,411 @@ class ASTBase(ASTBaseBase):
pass
-class ASTIdentifier(ASTBase):
+# Names
+################################################################################
- def __init__(self, name: str) ->None:
+class ASTIdentifier(ASTBase):
+ def __init__(self, name: str) -> None:
if not isinstance(name, str) or len(name) == 0:
raise AssertionError
self.name = sys.intern(name)
self.is_anonymous = name[0] == '@'
- def __eq__(self, other: object) ->bool:
+ # ASTBaseBase already implements this method,
+ # but specialising it here improves performance
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTIdentifier):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
- def __str__(self) ->str:
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.name)
+
+ def is_anon(self) -> bool:
+ return self.is_anonymous
+
+ def get_id(self, version: int) -> str:
+ if self.is_anonymous and version < 3:
+ raise NoOldIdError
+ if version == 1:
+ if self.name == 'size_t':
+ return 's'
+ else:
+ return self.name
+ if self.name == "std":
+ return 'St'
+ elif self.name[0] == "~":
+ # a destructor, just use an arbitrary version of dtors
+ return 'D0'
+ else:
+ if self.is_anonymous:
+ return 'Ut%d_%s' % (len(self.name) - 1, self.name[1:])
+ else:
+ return str(len(self.name)) + self.name
+
+ # and this is where we finally make a difference between __str__ and the display string
+
+ def __str__(self) -> str:
return self.name
+ def get_display_string(self) -> str:
+ return "[anonymous]" if self.is_anonymous else self.name
-class ASTNestedNameElement(ASTBase):
+ def describe_signature(self, signode: TextElement, mode: str, env: BuildEnvironment,
+ prefix: str, templateArgs: str, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.is_anonymous:
+ node = addnodes.desc_sig_name(text="[anonymous]")
+ else:
+ node = addnodes.desc_sig_name(self.name, self.name)
+ if mode == 'markType':
+ targetText = prefix + self.name + templateArgs
+ pnode = addnodes.pending_xref('', refdomain='cpp',
+ reftype='identifier',
+ reftarget=targetText, modname=None,
+ classname=None)
+ pnode['cpp:parent_key'] = symbol.get_lookup_key()
+ pnode += node
+ signode += pnode
+ elif mode == 'lastIsName':
+ nameNode = addnodes.desc_name()
+ nameNode += node
+ signode += nameNode
+ elif mode == 'noneIsName':
+ signode += node
+ elif mode == 'param':
+ node['classes'].append('sig-param')
+ signode += node
+ elif mode == 'udl':
+ # the target is 'operator""id' instead of just 'id'
+ assert len(prefix) == 0
+ assert len(templateArgs) == 0
+ assert not self.is_anonymous
+ targetText = 'operator""' + self.name
+ pnode = addnodes.pending_xref('', refdomain='cpp',
+ reftype='identifier',
+ reftarget=targetText, modname=None,
+ classname=None)
+ pnode['cpp:parent_key'] = symbol.get_lookup_key()
+ pnode += node
+ signode += pnode
+ else:
+ raise Exception('Unknown description mode: %s' % mode)
+
+ @property
+ def identifier(self) -> str:
+ warnings.warn(
+ '`ASTIdentifier.identifier` is deprecated, use `ASTIdentifier.name` instead',
+ DeprecationWarning, stacklevel=2,
+ )
+ return self.name
- def __init__(self, identOrOp: (ASTIdentifier | ASTOperator),
- templateArgs: (ASTTemplateArgs | None)) ->None:
+
+class ASTNestedNameElement(ASTBase):
+ def __init__(self, identOrOp: ASTIdentifier | ASTOperator,
+ templateArgs: ASTTemplateArgs | None) -> None:
self.identOrOp = identOrOp
self.templateArgs = templateArgs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNestedNameElement):
return NotImplemented
- return (self.identOrOp == other.identOrOp and self.templateArgs ==
- other.templateArgs)
+ return self.identOrOp == other.identOrOp and self.templateArgs == other.templateArgs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.identOrOp, self.templateArgs))
+ def is_operator(self) -> bool:
+ return False
-class ASTNestedName(ASTBase):
+ def get_id(self, version: int) -> str:
+ res = self.identOrOp.get_id(version)
+ if self.templateArgs:
+ res += self.templateArgs.get_id(version)
+ return res
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = transform(self.identOrOp)
+ if self.templateArgs:
+ res += transform(self.templateArgs)
+ return res
- def __init__(self, names: list[ASTNestedNameElement], templates: list[
- bool], rooted: bool) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, prefix: str, symbol: Symbol) -> None:
+ tArgs = str(self.templateArgs) if self.templateArgs is not None else ''
+ self.identOrOp.describe_signature(signode, mode, env, prefix, tArgs, symbol)
+ if self.templateArgs is not None:
+ self.templateArgs.describe_signature(signode, 'markType', env, symbol)
+
+
+class ASTNestedName(ASTBase):
+ def __init__(self, names: list[ASTNestedNameElement],
+ templates: list[bool], rooted: bool) -> None:
assert len(names) > 0
self.names = names
self.templates = templates
assert len(self.names) == len(self.templates)
self.rooted = rooted
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNestedName):
return NotImplemented
- return (self.names == other.names and self.templates == other.
- templates and self.rooted == other.rooted)
+ return (
+ self.names == other.names
+ and self.templates == other.templates
+ and self.rooted == other.rooted
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.names, self.templates, self.rooted))
+ @property
+ def name(self) -> ASTNestedName:
+ return self
+
+ def num_templates(self) -> int:
+ count = 0
+ for n in self.names:
+ if n.is_operator():
+ continue
+ if n.templateArgs:
+ count += 1
+ return count
+
+ def get_id(self, version: int, modifiers: str = '') -> str:
+ if version == 1:
+ tt = str(self)
+ if tt in _id_shorthands_v1:
+ return _id_shorthands_v1[tt]
+ else:
+ return '::'.join(n.get_id(version) for n in self.names)
+
+ res = []
+ if len(self.names) > 1 or len(modifiers) > 0:
+ res.append('N')
+ res.append(modifiers)
+ res.extend(n.get_id(version) for n in self.names)
+ if len(self.names) > 1 or len(modifiers) > 0:
+ res.append('E')
+ return ''.join(res)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.rooted:
+ res.append('')
+ for i in range(len(self.names)):
+ n = self.names[i]
+ if self.templates[i]:
+ res.append("template " + transform(n))
+ else:
+ res.append(transform(n))
+ return '::'.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ # just print the name part, with template args, not template params
+ if mode == 'noneIsName':
+ if self.rooted:
+ unreachable = "Can this happen?"
+ raise AssertionError(unreachable) # TODO
+ signode += nodes.Text('::')
+ for i in range(len(self.names)):
+ if i != 0:
+ unreachable = "Can this happen?"
+ raise AssertionError(unreachable) # TODO
+ signode += nodes.Text('::blah')
+ n = self.names[i]
+ if self.templates[i]:
+ unreachable = "Can this happen?"
+ raise AssertionError(unreachable) # TODO
+ signode += nodes.Text("template")
+ signode += nodes.Text(" ")
+ n.describe_signature(signode, mode, env, '', symbol)
+ elif mode == 'param':
+ assert not self.rooted, str(self)
+ assert len(self.names) == 1
+ assert not self.templates[0]
+ self.names[0].describe_signature(signode, 'param', env, '', symbol)
+ elif mode in ('markType', 'lastIsName', 'markName'):
+ # Each element should be a pending xref targeting the complete
+ # prefix. however, only the identifier part should be a link, such
+ # that template args can be a link as well.
+ # For 'lastIsName' we should also prepend template parameter lists.
+ templateParams: list[Any] = []
+ if mode == 'lastIsName':
+ assert symbol is not None
+ if symbol.declaration.templatePrefix is not None:
+ templateParams = symbol.declaration.templatePrefix.templates
+ iTemplateParams = 0
+ templateParamsPrefix = ''
+ prefix = ''
+ first = True
+ names = self.names[:-1] if mode == 'lastIsName' else self.names
+ # If lastIsName, then wrap all of the prefix in a desc_addname,
+ # else append directly to signode.
+ # NOTE: Breathe previously relied on the prefix being in the desc_addname node,
+ # so it can remove it in inner declarations.
+ dest = signode
+ if mode == 'lastIsName':
+ dest = addnodes.desc_addname()
+ if self.rooted:
+ prefix += '::'
+ if mode == 'lastIsName' and len(names) == 0:
+ signode += addnodes.desc_sig_punctuation('::', '::')
+ else:
+ dest += addnodes.desc_sig_punctuation('::', '::')
+ for i in range(len(names)):
+ nne = names[i]
+ template = self.templates[i]
+ if not first:
+ dest += addnodes.desc_sig_punctuation('::', '::')
+ prefix += '::'
+ if template:
+ dest += addnodes.desc_sig_keyword('template', 'template')
+ dest += addnodes.desc_sig_space()
+ first = False
+ txt_nne = str(nne)
+ if txt_nne != '':
+ if nne.templateArgs and iTemplateParams < len(templateParams):
+ templateParamsPrefix += str(templateParams[iTemplateParams])
+ iTemplateParams += 1
+ nne.describe_signature(dest, 'markType',
+ env, templateParamsPrefix + prefix, symbol)
+ prefix += txt_nne
+ if mode == 'lastIsName':
+ if len(self.names) > 1:
+ dest += addnodes.desc_sig_punctuation('::', '::')
+ signode += dest
+ if self.templates[-1]:
+ signode += addnodes.desc_sig_keyword('template', 'template')
+ signode += addnodes.desc_sig_space()
+ self.names[-1].describe_signature(signode, mode, env, '', symbol)
+ else:
+ raise Exception('Unknown description mode: %s' % mode)
+
+
+################################################################################
+# Expressions
+################################################################################
class ASTExpression(ASTBase):
- pass
+ def get_id(self, version: int) -> str:
+ raise NotImplementedError(repr(self))
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ raise NotImplementedError(repr(self))
+
+# Primary expressions
+################################################################################
class ASTLiteral(ASTExpression):
pass
class ASTPointerLiteral(ASTLiteral):
-
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
return isinstance(other, ASTPointerLiteral)
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash('nullptr')
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return 'nullptr'
-class ASTBooleanLiteral(ASTLiteral):
+ def get_id(self, version: int) -> str:
+ return 'LDnE'
- def __init__(self, value: bool) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('nullptr', 'nullptr')
+
+
+class ASTBooleanLiteral(ASTLiteral):
+ def __init__(self, value: bool) -> None:
self.value = value
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTBooleanLiteral):
return NotImplemented
return self.value == other.value
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.value)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.value:
+ return 'true'
+ else:
+ return 'false'
-class ASTNumberLiteral(ASTLiteral):
+ def get_id(self, version: int) -> str:
+ if self.value:
+ return 'L1E'
+ else:
+ return 'L0E'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword(str(self), str(self))
- def __init__(self, data: str) ->None:
+
+class ASTNumberLiteral(ASTLiteral):
+ def __init__(self, data: str) -> None:
self.data = data
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNumberLiteral):
return NotImplemented
return self.data == other.data
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.data)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return self.data
-class ASTStringLiteral(ASTLiteral):
+ def get_id(self, version: int) -> str:
+ # TODO: floats should be mangled by writing the hex of the binary representation
+ return "L%sE" % self.data.replace("'", "")
- def __init__(self, data: str) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_literal_number(self.data, self.data)
+
+
+class ASTStringLiteral(ASTLiteral):
+ def __init__(self, data: str) -> None:
self.data = data
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTStringLiteral):
return NotImplemented
return self.data == other.data
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.data)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return self.data
-class ASTCharLiteral(ASTLiteral):
+ def get_id(self, version: int) -> str:
+ # note: the length is not really correct with escaping
+ return "LA%d_KcE" % (len(self.data) - 2)
- def __init__(self, prefix: str, data: str) ->None:
- self.prefix = prefix
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_literal_string(self.data, self.data)
+
+
+class ASTCharLiteral(ASTLiteral):
+ def __init__(self, prefix: str, data: str) -> None:
+ self.prefix = prefix # may be None when no prefix
self.data = data
assert prefix in _id_char_from_prefix
self.type = _id_char_from_prefix[prefix]
@@ -147,639 +455,1490 @@ class ASTCharLiteral(ASTLiteral):
else:
raise UnsupportedMultiCharacterCharLiteral(decoded)
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTCharLiteral):
return NotImplemented
- return self.prefix == other.prefix and self.value == other.value
+ return (
+ self.prefix == other.prefix
+ and self.value == other.value
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.prefix, self.value))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.prefix is None:
+ return "'" + self.data + "'"
+ else:
+ return self.prefix + "'" + self.data + "'"
-class ASTUserDefinedLiteral(ASTLiteral):
+ def get_id(self, version: int) -> str:
+ # TODO: the ID should be have L E around it
+ return self.type + str(self.value)
- def __init__(self, literal: ASTLiteral, ident: ASTIdentifier) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ if self.prefix is not None:
+ signode += addnodes.desc_sig_keyword(self.prefix, self.prefix)
+ txt = "'" + self.data + "'"
+ signode += addnodes.desc_sig_literal_char(txt, txt)
+
+
+class ASTUserDefinedLiteral(ASTLiteral):
+ def __init__(self, literal: ASTLiteral, ident: ASTIdentifier) -> None:
self.literal = literal
self.ident = ident
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTUserDefinedLiteral):
return NotImplemented
return self.literal == other.literal and self.ident == other.ident
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.literal, self.ident))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.literal) + transform(self.ident)
-class ASTThisLiteral(ASTExpression):
+ def get_id(self, version: int) -> str:
+ # mangle as if it was a function call: ident(literal)
+ return f'clL_Zli{self.ident.get_id(version)}E{self.literal.get_id(version)}E'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.literal.describe_signature(signode, mode, env, symbol)
+ self.ident.describe_signature(signode, "udl", env, "", "", symbol)
+
+
+################################################################################
- def __eq__(self, other: object) ->bool:
+class ASTThisLiteral(ASTExpression):
+ def __eq__(self, other: object) -> bool:
return isinstance(other, ASTThisLiteral)
- def __hash__(self) ->int:
- return hash('this')
+ def __hash__(self) -> int:
+ return hash("this")
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return "this"
-class ASTFoldExpr(ASTExpression):
+ def get_id(self, version: int) -> str:
+ return "fpT"
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('this', 'this')
- def __init__(self, leftExpr: (ASTExpression | None), op: str, rightExpr:
- (ASTExpression | None)) ->None:
+
+class ASTFoldExpr(ASTExpression):
+ def __init__(self, leftExpr: ASTExpression | None,
+ op: str, rightExpr: ASTExpression | None) -> None:
assert leftExpr is not None or rightExpr is not None
self.leftExpr = leftExpr
self.op = op
self.rightExpr = rightExpr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTFoldExpr):
return NotImplemented
- return (self.leftExpr == other.leftExpr and self.op == other.op and
- self.rightExpr == other.rightExpr)
+ return (
+ self.leftExpr == other.leftExpr
+ and self.op == other.op
+ and self.rightExpr == other.rightExpr
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.leftExpr, self.op, self.rightExpr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['(']
+ if self.leftExpr:
+ res.append(transform(self.leftExpr))
+ res.append(' ')
+ res.append(self.op)
+ res.append(' ')
+ res.append('...')
+ if self.rightExpr:
+ res.append(' ')
+ res.append(self.op)
+ res.append(' ')
+ res.append(transform(self.rightExpr))
+ res.append(')')
+ return ''.join(res)
+
+ def get_id(self, version: int) -> str:
+ assert version >= 3
+ if version == 3:
+ return str(self)
+ # https://github.com/itanium-cxx-abi/cxx-abi/pull/67
+ res = []
+ if self.leftExpr is None: # (... op expr)
+ res.append('fl')
+ elif self.rightExpr is None: # (expr op ...)
+ res.append('fr')
+ else: # (expr op ... op expr)
+ # we don't check where the parameter pack is,
+ # we just always call this a binary left fold
+ res.append('fL')
+ res.append(_id_operator_v2[self.op])
+ if self.leftExpr:
+ res.append(self.leftExpr.get_id(version))
+ if self.rightExpr:
+ res.append(self.rightExpr.get_id(version))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ if self.leftExpr:
+ self.leftExpr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_operator(self.op, self.op)
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ if self.rightExpr:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_operator(self.op, self.op)
+ signode += addnodes.desc_sig_space()
+ self.rightExpr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
-class ASTParenExpr(ASTExpression):
- def __init__(self, expr: ASTExpression) ->None:
+class ASTParenExpr(ASTExpression):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTParenExpr):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '(' + transform(self.expr) + ')'
+
+ def get_id(self, version: int) -> str:
+ return self.expr.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.expr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
-class ASTIdExpression(ASTExpression):
- def __init__(self, name: ASTNestedName) ->None:
+class ASTIdExpression(ASTExpression):
+ def __init__(self, name: ASTNestedName) -> None:
+ # note: this class is basically to cast a nested name as an expression
self.name = name
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTIdExpression):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.name)
+
+ def get_id(self, version: int) -> str:
+ return self.name.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.name.describe_signature(signode, mode, env, symbol)
+
+
+# Postfix expressions
+################################################################################
class ASTPostfixOp(ASTBase):
- pass
+ def get_id(self, idPrefix: str, version: int) -> str:
+ raise NotImplementedError(repr(self))
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ raise NotImplementedError(repr(self))
-class ASTPostfixArray(ASTPostfixOp):
- def __init__(self, expr: ASTExpression) ->None:
+class ASTPostfixArray(ASTPostfixOp):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixArray):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '[' + transform(self.expr) + ']'
+
+ def get_id(self, idPrefix: str, version: int) -> str:
+ return 'ix' + idPrefix + self.expr.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_punctuation('[', '[')
+ self.expr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(']', ']')
-class ASTPostfixMember(ASTPostfixOp):
- def __init__(self, name: ASTNestedName) ->None:
+class ASTPostfixMember(ASTPostfixOp):
+ def __init__(self, name: ASTNestedName) -> None:
self.name = name
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixMember):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '.' + transform(self.name)
+
+ def get_id(self, idPrefix: str, version: int) -> str:
+ return 'dt' + idPrefix + self.name.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_punctuation('.', '.')
+ self.name.describe_signature(signode, 'noneIsName', env, symbol)
-class ASTPostfixMemberOfPointer(ASTPostfixOp):
- def __init__(self, name: ASTNestedName) ->None:
+class ASTPostfixMemberOfPointer(ASTPostfixOp):
+ def __init__(self, name: ASTNestedName) -> None:
self.name = name
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixMemberOfPointer):
return NotImplemented
return self.name == other.name
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '->' + transform(self.name)
-class ASTPostfixInc(ASTPostfixOp):
+ def get_id(self, idPrefix: str, version: int) -> str:
+ return 'pt' + idPrefix + self.name.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_operator('->', '->')
+ self.name.describe_signature(signode, 'noneIsName', env, symbol)
- def __eq__(self, other: object) ->bool:
+
+class ASTPostfixInc(ASTPostfixOp):
+ def __eq__(self, other: object) -> bool:
return isinstance(other, ASTPostfixInc)
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash('++')
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '++'
-class ASTPostfixDec(ASTPostfixOp):
+ def get_id(self, idPrefix: str, version: int) -> str:
+ return 'pp' + idPrefix
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_operator('++', '++')
- def __eq__(self, other: object) ->bool:
+
+class ASTPostfixDec(ASTPostfixOp):
+ def __eq__(self, other: object) -> bool:
return isinstance(other, ASTPostfixDec)
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash('--')
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return '--'
-class ASTPostfixCallExpr(ASTPostfixOp):
+ def get_id(self, idPrefix: str, version: int) -> str:
+ return 'mm' + idPrefix
- def __init__(self, lst: (ASTParenExprList | ASTBracedInitList)) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_operator('--', '--')
+
+
+class ASTPostfixCallExpr(ASTPostfixOp):
+ def __init__(self, lst: ASTParenExprList | ASTBracedInitList) -> None:
self.lst = lst
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixCallExpr):
return NotImplemented
return self.lst == other.lst
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.lst)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.lst)
-class ASTPostfixExpr(ASTExpression):
+ def get_id(self, idPrefix: str, version: int) -> str:
+ return ''.join([
+ 'cl',
+ idPrefix,
+ *(e.get_id(version) for e in self.lst.exprs),
+ 'E',
+ ])
- def __init__(self, prefix: ASTType, postFixes: list[ASTPostfixOp]) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.lst.describe_signature(signode, mode, env, symbol)
+
+
+class ASTPostfixExpr(ASTExpression):
+ def __init__(self, prefix: ASTType, postFixes: list[ASTPostfixOp]) -> None:
self.prefix = prefix
self.postFixes = postFixes
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPostfixExpr):
return NotImplemented
- return (self.prefix == other.prefix and self.postFixes == other.
- postFixes)
+ return self.prefix == other.prefix and self.postFixes == other.postFixes
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.prefix, self.postFixes))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return ''.join([transform(self.prefix), *(transform(p) for p in self.postFixes)])
+
+ def get_id(self, version: int) -> str:
+ id = self.prefix.get_id(version)
+ for p in self.postFixes:
+ id = p.get_id(id, version)
+ return id
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.prefix.describe_signature(signode, mode, env, symbol)
+ for p in self.postFixes:
+ p.describe_signature(signode, mode, env, symbol)
-class ASTExplicitCast(ASTExpression):
- def __init__(self, cast: str, typ: ASTType, expr: ASTExpression) ->None:
+class ASTExplicitCast(ASTExpression):
+ def __init__(self, cast: str, typ: ASTType, expr: ASTExpression) -> None:
assert cast in _id_explicit_cast
self.cast = cast
self.typ = typ
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTExplicitCast):
return NotImplemented
- return (self.cast == other.cast and self.typ == other.typ and self.
- expr == other.expr)
+ return self.cast == other.cast and self.typ == other.typ and self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.cast, self.typ, self.expr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = [self.cast]
+ res.append('<')
+ res.append(transform(self.typ))
+ res.append('>(')
+ res.append(transform(self.expr))
+ res.append(')')
+ return ''.join(res)
+
+ def get_id(self, version: int) -> str:
+ return (_id_explicit_cast[self.cast] +
+ self.typ.get_id(version) +
+ self.expr.get_id(version))
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword(self.cast, self.cast)
+ signode += addnodes.desc_sig_punctuation('<', '<')
+ self.typ.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation('>', '>')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.expr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
-class ASTTypeId(ASTExpression):
- def __init__(self, typeOrExpr: (ASTType | ASTExpression), isType: bool
- ) ->None:
+class ASTTypeId(ASTExpression):
+ def __init__(self, typeOrExpr: ASTType | ASTExpression, isType: bool) -> None:
self.typeOrExpr = typeOrExpr
self.isType = isType
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTypeId):
return NotImplemented
- return (self.typeOrExpr == other.typeOrExpr and self.isType ==
- other.isType)
+ return self.typeOrExpr == other.typeOrExpr and self.isType == other.isType
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.typeOrExpr, self.isType))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return 'typeid(' + transform(self.typeOrExpr) + ')'
-class ASTUnaryOpExpr(ASTExpression):
+ def get_id(self, version: int) -> str:
+ prefix = 'ti' if self.isType else 'te'
+ return prefix + self.typeOrExpr.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('typeid', 'typeid')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.typeOrExpr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+
+
+# Unary expressions
+################################################################################
- def __init__(self, op: str, expr: ASTExpression) ->None:
+class ASTUnaryOpExpr(ASTExpression):
+ def __init__(self, op: str, expr: ASTExpression) -> None:
self.op = op
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTUnaryOpExpr):
return NotImplemented
return self.op == other.op and self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.op, self.expr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.op[0] in 'cn':
+ return self.op + " " + transform(self.expr)
+ else:
+ return self.op + transform(self.expr)
-class ASTSizeofParamPack(ASTExpression):
+ def get_id(self, version: int) -> str:
+ return _id_operator_unary_v2[self.op] + self.expr.get_id(version)
- def __init__(self, identifier: ASTIdentifier) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ if self.op[0] in 'cn':
+ signode += addnodes.desc_sig_keyword(self.op, self.op)
+ signode += addnodes.desc_sig_space()
+ else:
+ signode += addnodes.desc_sig_operator(self.op, self.op)
+ self.expr.describe_signature(signode, mode, env, symbol)
+
+
+class ASTSizeofParamPack(ASTExpression):
+ def __init__(self, identifier: ASTIdentifier) -> None:
self.identifier = identifier
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTSizeofParamPack):
return NotImplemented
return self.identifier == other.identifier
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.identifier)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return "sizeof...(" + transform(self.identifier) + ")"
+
+ def get_id(self, version: int) -> str:
+ return 'sZ' + self.identifier.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('sizeof', 'sizeof')
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.identifier.describe_signature(signode, 'markType', env,
+ symbol=symbol, prefix="", templateArgs="")
+ signode += addnodes.desc_sig_punctuation(')', ')')
-class ASTSizeofType(ASTExpression):
- def __init__(self, typ: ASTType) ->None:
+class ASTSizeofType(ASTExpression):
+ def __init__(self, typ: ASTType) -> None:
self.typ = typ
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTSizeofType):
return NotImplemented
return self.typ == other.typ
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.typ)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return "sizeof(" + transform(self.typ) + ")"
+
+ def get_id(self, version: int) -> str:
+ return 'st' + self.typ.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('sizeof', 'sizeof')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.typ.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
-class ASTSizeofExpr(ASTExpression):
- def __init__(self, expr: ASTExpression) ->None:
+class ASTSizeofExpr(ASTExpression):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTSizeofExpr):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return "sizeof " + transform(self.expr)
-class ASTAlignofExpr(ASTExpression):
+ def get_id(self, version: int) -> str:
+ return 'sz' + self.expr.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('sizeof', 'sizeof')
+ signode += addnodes.desc_sig_space()
+ self.expr.describe_signature(signode, mode, env, symbol)
- def __init__(self, typ: ASTType) ->None:
+
+class ASTAlignofExpr(ASTExpression):
+ def __init__(self, typ: ASTType) -> None:
self.typ = typ
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTAlignofExpr):
return NotImplemented
return self.typ == other.typ
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.typ)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return "alignof(" + transform(self.typ) + ")"
-class ASTNoexceptExpr(ASTExpression):
+ def get_id(self, version: int) -> str:
+ return 'at' + self.typ.get_id(version)
- def __init__(self, expr: ASTExpression) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('alignof', 'alignof')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.typ.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+
+
+class ASTNoexceptExpr(ASTExpression):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNoexceptExpr):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return 'noexcept(' + transform(self.expr) + ')'
-class ASTNewExpr(ASTExpression):
+ def get_id(self, version: int) -> str:
+ return 'nx' + self.expr.get_id(version)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('noexcept', 'noexcept')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.expr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+
+class ASTNewExpr(ASTExpression):
def __init__(self, rooted: bool, isNewTypeId: bool, typ: ASTType,
- initList: (ASTParenExprList | ASTBracedInitList)) ->None:
+ initList: ASTParenExprList | ASTBracedInitList) -> None:
self.rooted = rooted
self.isNewTypeId = isNewTypeId
self.typ = typ
self.initList = initList
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNewExpr):
return NotImplemented
- return (self.rooted == other.rooted and self.isNewTypeId == other.
- isNewTypeId and self.typ == other.typ and self.initList ==
- other.initList)
-
- def __hash__(self) ->int:
+ return (
+ self.rooted == other.rooted
+ and self.isNewTypeId == other.isNewTypeId
+ and self.typ == other.typ
+ and self.initList == other.initList
+ )
+
+ def __hash__(self) -> int:
return hash((self.rooted, self.isNewTypeId, self.typ, self.initList))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.rooted:
+ res.append('::')
+ res.append('new ')
+ # TODO: placement
+ if self.isNewTypeId:
+ res.append(transform(self.typ))
+ else:
+ raise AssertionError
+ if self.initList is not None:
+ res.append(transform(self.initList))
+ return ''.join(res)
+
+ def get_id(self, version: int) -> str:
+ # the array part will be in the type mangling, so na is not used
+ res = ['nw']
+ # TODO: placement
+ res.append('_')
+ res.append(self.typ.get_id(version))
+ if self.initList is not None:
+ res.append(self.initList.get_id(version))
+ else:
+ res.append('E')
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ if self.rooted:
+ signode += addnodes.desc_sig_punctuation('::', '::')
+ signode += addnodes.desc_sig_keyword('new', 'new')
+ signode += addnodes.desc_sig_space()
+ # TODO: placement
+ if self.isNewTypeId:
+ self.typ.describe_signature(signode, mode, env, symbol)
+ else:
+ raise AssertionError
+ if self.initList is not None:
+ self.initList.describe_signature(signode, mode, env, symbol)
+
class ASTDeleteExpr(ASTExpression):
-
- def __init__(self, rooted: bool, array: bool, expr: ASTExpression) ->None:
+ def __init__(self, rooted: bool, array: bool, expr: ASTExpression) -> None:
self.rooted = rooted
self.array = array
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeleteExpr):
return NotImplemented
- return (self.rooted == other.rooted and self.array == other.array and
- self.expr == other.expr)
+ return (
+ self.rooted == other.rooted
+ and self.array == other.array
+ and self.expr == other.expr
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.rooted, self.array, self.expr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.rooted:
+ res.append('::')
+ res.append('delete ')
+ if self.array:
+ res.append('[] ')
+ res.append(transform(self.expr))
+ return ''.join(res)
+
+ def get_id(self, version: int) -> str:
+ if self.array:
+ id = "da"
+ else:
+ id = "dl"
+ return id + self.expr.get_id(version)
-class ASTCastExpr(ASTExpression):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ if self.rooted:
+ signode += addnodes.desc_sig_punctuation('::', '::')
+ signode += addnodes.desc_sig_keyword('delete', 'delete')
+ signode += addnodes.desc_sig_space()
+ if self.array:
+ signode += addnodes.desc_sig_punctuation('[]', '[]')
+ signode += addnodes.desc_sig_space()
+ self.expr.describe_signature(signode, mode, env, symbol)
- def __init__(self, typ: ASTType, expr: ASTExpression) ->None:
+
+# Other expressions
+################################################################################
+
+class ASTCastExpr(ASTExpression):
+ def __init__(self, typ: ASTType, expr: ASTExpression) -> None:
self.typ = typ
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTCastExpr):
return NotImplemented
- return self.typ == other.typ and self.expr == other.expr
+ return (
+ self.typ == other.typ
+ and self.expr == other.expr
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.typ, self.expr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['(']
+ res.append(transform(self.typ))
+ res.append(')')
+ res.append(transform(self.expr))
+ return ''.join(res)
-class ASTBinOpExpr(ASTExpression):
+ def get_id(self, version: int) -> str:
+ return 'cv' + self.typ.get_id(version) + self.expr.get_id(version)
- def __init__(self, exprs: list[ASTExpression], ops: list[str]) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.typ.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+ self.expr.describe_signature(signode, mode, env, symbol)
+
+
+class ASTBinOpExpr(ASTExpression):
+ def __init__(self, exprs: list[ASTExpression], ops: list[str]) -> None:
assert len(exprs) > 0
assert len(exprs) == len(ops) + 1
self.exprs = exprs
self.ops = ops
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTBinOpExpr):
return NotImplemented
- return self.exprs == other.exprs and self.ops == other.ops
+ return (
+ self.exprs == other.exprs
+ and self.ops == other.ops
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.exprs, self.ops))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.exprs[0]))
+ for i in range(1, len(self.exprs)):
+ res.append(' ')
+ res.append(self.ops[i - 1])
+ res.append(' ')
+ res.append(transform(self.exprs[i]))
+ return ''.join(res)
+
+ def get_id(self, version: int) -> str:
+ assert version >= 2
+ res = []
+ for i in range(len(self.ops)):
+ res.append(_id_operator_v2[self.ops[i]])
+ res.append(self.exprs[i].get_id(version))
+ res.append(self.exprs[-1].get_id(version))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.exprs[0].describe_signature(signode, mode, env, symbol)
+ for i in range(1, len(self.exprs)):
+ signode += addnodes.desc_sig_space()
+ op = self.ops[i - 1]
+ if ord(op[0]) >= ord('a') and ord(op[0]) <= ord('z'):
+ signode += addnodes.desc_sig_keyword(op, op)
+ else:
+ signode += addnodes.desc_sig_operator(op, op)
+ signode += addnodes.desc_sig_space()
+ self.exprs[i].describe_signature(signode, mode, env, symbol)
-class ASTConditionalExpr(ASTExpression):
+class ASTConditionalExpr(ASTExpression):
def __init__(self, ifExpr: ASTExpression, thenExpr: ASTExpression,
- elseExpr: ASTExpression) ->None:
+ elseExpr: ASTExpression) -> None:
self.ifExpr = ifExpr
self.thenExpr = thenExpr
self.elseExpr = elseExpr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTConditionalExpr):
return NotImplemented
- return (self.ifExpr == other.ifExpr and self.thenExpr == other.
- thenExpr and self.elseExpr == other.elseExpr)
+ return (
+ self.ifExpr == other.ifExpr
+ and self.thenExpr == other.thenExpr
+ and self.elseExpr == other.elseExpr
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.ifExpr, self.thenExpr, self.elseExpr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.ifExpr))
+ res.append(' ? ')
+ res.append(transform(self.thenExpr))
+ res.append(' : ')
+ res.append(transform(self.elseExpr))
+ return ''.join(res)
+
+ def get_id(self, version: int) -> str:
+ assert version >= 2
+ res = []
+ res.append(_id_operator_v2['?'])
+ res.append(self.ifExpr.get_id(version))
+ res.append(self.thenExpr.get_id(version))
+ res.append(self.elseExpr.get_id(version))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.ifExpr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_operator('?', '?')
+ signode += addnodes.desc_sig_space()
+ self.thenExpr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_operator(':', ':')
+ signode += addnodes.desc_sig_space()
+ self.elseExpr.describe_signature(signode, mode, env, symbol)
-class ASTBracedInitList(ASTBase):
+class ASTBracedInitList(ASTBase):
def __init__(self, exprs: list[ASTExpression | ASTBracedInitList],
- trailingComma: bool) ->None:
+ trailingComma: bool) -> None:
self.exprs = exprs
self.trailingComma = trailingComma
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTBracedInitList):
return NotImplemented
- return (self.exprs == other.exprs and self.trailingComma == other.
- trailingComma)
+ return self.exprs == other.exprs and self.trailingComma == other.trailingComma
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.exprs, self.trailingComma))
+ def get_id(self, version: int) -> str:
+ return "il%sE" % ''.join(e.get_id(version) for e in self.exprs)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ exprs = ', '.join(transform(e) for e in self.exprs)
+ trailingComma = ',' if self.trailingComma else ''
+ return f'{{{exprs}{trailingComma}}}'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('{', '{')
+ first = True
+ for e in self.exprs:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ else:
+ first = False
+ e.describe_signature(signode, mode, env, symbol)
+ if self.trailingComma:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_punctuation('}', '}')
-class ASTAssignmentExpr(ASTExpression):
- def __init__(self, leftExpr: ASTExpression, op: str, rightExpr: (
- ASTExpression | ASTBracedInitList)) ->None:
+class ASTAssignmentExpr(ASTExpression):
+ def __init__(self, leftExpr: ASTExpression, op: str,
+ rightExpr: ASTExpression | ASTBracedInitList) -> None:
self.leftExpr = leftExpr
self.op = op
self.rightExpr = rightExpr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTAssignmentExpr):
return NotImplemented
- return (self.leftExpr == other.leftExpr and self.op == other.op and
- self.rightExpr == other.rightExpr)
+ return (
+ self.leftExpr == other.leftExpr
+ and self.op == other.op
+ and self.rightExpr == other.rightExpr
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.leftExpr, self.op, self.rightExpr))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.leftExpr))
+ res.append(' ')
+ res.append(self.op)
+ res.append(' ')
+ res.append(transform(self.rightExpr))
+ return ''.join(res)
+
+ def get_id(self, version: int) -> str:
+ # we end up generating the ID from left to right, instead of right to left
+ res = []
+ res.append(_id_operator_v2[self.op])
+ res.append(self.leftExpr.get_id(version))
+ res.append(self.rightExpr.get_id(version))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.leftExpr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_space()
+ if ord(self.op[0]) >= ord('a') and ord(self.op[0]) <= ord('z'):
+ signode += addnodes.desc_sig_keyword(self.op, self.op)
+ else:
+ signode += addnodes.desc_sig_operator(self.op, self.op)
+ signode += addnodes.desc_sig_space()
+ self.rightExpr.describe_signature(signode, mode, env, symbol)
-class ASTCommaExpr(ASTExpression):
- def __init__(self, exprs: list[ASTExpression]) ->None:
+class ASTCommaExpr(ASTExpression):
+ def __init__(self, exprs: list[ASTExpression]) -> None:
assert len(exprs) > 0
self.exprs = exprs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTCommaExpr):
return NotImplemented
return self.exprs == other.exprs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.exprs)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return ', '.join(transform(e) for e in self.exprs)
-class ASTFallbackExpr(ASTExpression):
+ def get_id(self, version: int) -> str:
+ id_ = _id_operator_v2[',']
+ res = []
+ for i in range(len(self.exprs) - 1):
+ res.append(id_)
+ res.append(self.exprs[i].get_id(version))
+ res.append(self.exprs[-1].get_id(version))
+ return ''.join(res)
- def __init__(self, expr: str) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.exprs[0].describe_signature(signode, mode, env, symbol)
+ for i in range(1, len(self.exprs)):
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ self.exprs[i].describe_signature(signode, mode, env, symbol)
+
+
+class ASTFallbackExpr(ASTExpression):
+ def __init__(self, expr: str) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTFallbackExpr):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return self.expr
+
+ def get_id(self, version: int) -> str:
+ return str(self.expr)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += nodes.literal(self.expr, self.expr)
+
+
+################################################################################
+# Types
+################################################################################
+
+# Things for ASTNestedName
+################################################################################
class ASTOperator(ASTBase):
is_anonymous: ClassVar[Literal[False]] = False
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
raise NotImplementedError(repr(self))
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
raise NotImplementedError(repr(self))
- def _describe_identifier(self, signode: TextElement, identnode:
- TextElement, env: BuildEnvironment, symbol: Symbol) ->None:
+ def is_anon(self) -> bool:
+ return self.is_anonymous
+
+ def is_operator(self) -> bool:
+ return True
+
+ def get_id(self, version: int) -> str:
+ raise NotImplementedError
+
+ def _describe_identifier(self, signode: TextElement, identnode: TextElement,
+ env: BuildEnvironment, symbol: Symbol) -> None:
"""Render the prefix into signode, and the last part into identnode."""
- pass
+ raise NotImplementedError
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, prefix: str, templateArgs: str,
+ symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if mode == 'lastIsName':
+ mainName = addnodes.desc_name()
+ self._describe_identifier(mainName, mainName, env, symbol)
+ signode += mainName
+ elif mode == 'markType':
+ targetText = prefix + str(self) + templateArgs
+ pnode = addnodes.pending_xref('', refdomain='cpp',
+ reftype='identifier',
+ reftarget=targetText, modname=None,
+ classname=None)
+ pnode['cpp:parent_key'] = symbol.get_lookup_key()
+ # Render the identifier part, but collapse it into a string
+ # and make that the a link to this operator.
+ # E.g., if it is 'operator SomeType', then 'SomeType' becomes
+ # a link to the operator, not to 'SomeType'.
+ container = nodes.literal()
+ self._describe_identifier(signode, container, env, symbol)
+ txt = container.astext()
+ pnode += addnodes.desc_name(txt, txt)
+ signode += pnode
+ else:
+ addName = addnodes.desc_addname()
+ self._describe_identifier(addName, addName, env, symbol)
+ signode += addName
class ASTOperatorBuildIn(ASTOperator):
-
- def __init__(self, op: str) ->None:
+ def __init__(self, op: str) -> None:
self.op = op
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTOperatorBuildIn):
return NotImplemented
return self.op == other.op
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.op)
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ ids = _id_operator_v1
+ if self.op not in ids:
+ raise NoOldIdError
+ else:
+ ids = _id_operator_v2
+ if self.op not in ids:
+ raise Exception('Internal error: Built-in operator "%s" can not '
+ 'be mapped to an id.' % self.op)
+ return ids[self.op]
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.op in ('new', 'new[]', 'delete', 'delete[]') or self.op[0] in "abcnox":
+ return 'operator ' + self.op
+ else:
+ return 'operator' + self.op
+
+ def _describe_identifier(self, signode: TextElement, identnode: TextElement,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('operator', 'operator')
+ if self.op in ('new', 'new[]', 'delete', 'delete[]') or self.op[0] in "abcnox":
+ signode += addnodes.desc_sig_space()
+ identnode += addnodes.desc_sig_operator(self.op, self.op)
-class ASTOperatorLiteral(ASTOperator):
- def __init__(self, identifier: ASTIdentifier) ->None:
+class ASTOperatorLiteral(ASTOperator):
+ def __init__(self, identifier: ASTIdentifier) -> None:
self.identifier = identifier
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTOperatorLiteral):
return NotImplemented
return self.identifier == other.identifier
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.identifier)
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return 'li' + self.identifier.get_id(version)
-class ASTOperatorType(ASTOperator):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return 'operator""' + transform(self.identifier)
+
+ def _describe_identifier(self, signode: TextElement, identnode: TextElement,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('operator', 'operator')
+ signode += addnodes.desc_sig_literal_string('""', '""')
+ self.identifier.describe_signature(identnode, 'markType', env, '', '', symbol)
- def __init__(self, type: ASTType) ->None:
+
+class ASTOperatorType(ASTOperator):
+ def __init__(self, type: ASTType) -> None:
self.type = type
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTOperatorType):
return NotImplemented
return self.type == other.type
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.type)
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ return 'castto-%s-operator' % self.type.get_id(version)
+ else:
+ return 'cv' + self.type.get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return f'operator {transform(self.type)}'
+
+ def get_name_no_template(self) -> str:
+ return str(self)
+
+ def _describe_identifier(self, signode: TextElement, identnode: TextElement,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('operator', 'operator')
+ signode += addnodes.desc_sig_space()
+ self.type.describe_signature(identnode, 'markType', env, symbol)
-class ASTTemplateArgConstant(ASTBase):
- def __init__(self, value: ASTExpression) ->None:
+class ASTTemplateArgConstant(ASTBase):
+ def __init__(self, value: ASTExpression) -> None:
self.value = value
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateArgConstant):
return NotImplemented
return self.value == other.value
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.value)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.value)
-class ASTTemplateArgs(ASTBase):
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ return str(self).replace(' ', '-')
+ if version == 2:
+ return 'X' + str(self) + 'E'
+ return 'X' + self.value.get_id(version) + 'E'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.value.describe_signature(signode, mode, env, symbol)
+
+class ASTTemplateArgs(ASTBase):
def __init__(self, args: list[ASTType | ASTTemplateArgConstant],
- packExpansion: bool) ->None:
+ packExpansion: bool) -> None:
assert args is not None
self.args = args
self.packExpansion = packExpansion
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateArgs):
return NotImplemented
- return (self.args == other.args and self.packExpansion == other.
- packExpansion)
+ return self.args == other.args and self.packExpansion == other.packExpansion
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.args, self.packExpansion))
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ res = []
+ res.append(':')
+ res.append('.'.join(a.get_id(version) for a in self.args))
+ res.append(':')
+ return ''.join(res)
+
+ res = []
+ res.append('I')
+ if len(self.args) > 0:
+ for a in self.args[:-1]:
+ res.append(a.get_id(version))
+ if self.packExpansion:
+ res.append('J')
+ res.append(self.args[-1].get_id(version))
+ if self.packExpansion:
+ res.append('E')
+ res.append('E')
+ return ''.join(res)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ', '.join(transform(a) for a in self.args)
+ if self.packExpansion:
+ res += '...'
+ return '<' + res + '>'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('<', '<')
+ first = True
+ for a in self.args:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ first = False
+ a.describe_signature(signode, 'markType', env, symbol=symbol)
+ if self.packExpansion:
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ signode += addnodes.desc_sig_punctuation('>', '>')
+
+
+# Main part of declarations
+################################################################################
class ASTTrailingTypeSpec(ASTBase):
- pass
+ def get_id(self, version: int) -> str:
+ raise NotImplementedError(repr(self))
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ raise NotImplementedError(repr(self))
-class ASTTrailingTypeSpecFundamental(ASTTrailingTypeSpec):
- def __init__(self, names: list[str], canonNames: list[str]) ->None:
+class ASTTrailingTypeSpecFundamental(ASTTrailingTypeSpec):
+ def __init__(self, names: list[str], canonNames: list[str]) -> None:
assert len(names) != 0
assert len(names) == len(canonNames), (names, canonNames)
self.names = names
+ # the canonical name list is for ID lookup
self.canonNames = canonNames
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTrailingTypeSpecFundamental):
return NotImplemented
- return (self.names == other.names and self.canonNames == other.
- canonNames)
+ return self.names == other.names and self.canonNames == other.canonNames
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.names, self.canonNames))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return ' '.join(self.names)
+
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ res = []
+ for a in self.canonNames:
+ if a in _id_fundamental_v1:
+ res.append(_id_fundamental_v1[a])
+ else:
+ res.append(a)
+ return '-'.join(res)
+
+ txt = ' '.join(self.canonNames)
+ if txt not in _id_fundamental_v2:
+ raise Exception(
+ 'Semi-internal error: Fundamental type "%s" can not be mapped '
+ 'to an ID. Is it a true fundamental type? If not so, the '
+ 'parser should have rejected it.' % txt)
+ return _id_fundamental_v2[txt]
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ first = True
+ for n in self.names:
+ if not first:
+ signode += addnodes.desc_sig_space()
+ else:
+ first = False
+ signode += addnodes.desc_sig_keyword_type(n, n)
-class ASTTrailingTypeSpecDecltypeAuto(ASTTrailingTypeSpec):
- def __eq__(self, other: object) ->bool:
+class ASTTrailingTypeSpecDecltypeAuto(ASTTrailingTypeSpec):
+ def __eq__(self, other: object) -> bool:
return isinstance(other, ASTTrailingTypeSpecDecltypeAuto)
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash('decltype(auto)')
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return 'decltype(auto)'
-class ASTTrailingTypeSpecDecltype(ASTTrailingTypeSpec):
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return 'Dc'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('decltype', 'decltype')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ signode += addnodes.desc_sig_keyword('auto', 'auto')
+ signode += addnodes.desc_sig_punctuation(')', ')')
- def __init__(self, expr: ASTExpression) ->None:
+
+class ASTTrailingTypeSpecDecltype(ASTTrailingTypeSpec):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTrailingTypeSpecDecltype):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return 'decltype(' + transform(self.expr) + ')'
-class ASTTrailingTypeSpecName(ASTTrailingTypeSpec):
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return 'DT' + self.expr.get_id(version) + "E"
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('decltype', 'decltype')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.expr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+
+
+class ASTTrailingTypeSpecName(ASTTrailingTypeSpec):
def __init__(self, prefix: str, nestedName: ASTNestedName,
- placeholderType: (str | None)) ->None:
+ placeholderType: str | None) -> None:
self.prefix = prefix
self.nestedName = nestedName
self.placeholderType = placeholderType
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTrailingTypeSpecName):
return NotImplemented
- return (self.prefix == other.prefix and self.nestedName == other.
- nestedName and self.placeholderType == other.placeholderType)
+ return (
+ self.prefix == other.prefix
+ and self.nestedName == other.nestedName
+ and self.placeholderType == other.placeholderType
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.prefix, self.nestedName, self.placeholderType))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.nestedName
+
+ def get_id(self, version: int) -> str:
+ return self.nestedName.get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.prefix:
+ res.append(self.prefix)
+ res.append(' ')
+ res.append(transform(self.nestedName))
+ if self.placeholderType is not None:
+ res.append(' ')
+ res.append(self.placeholderType)
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ if self.prefix:
+ signode += addnodes.desc_sig_keyword(self.prefix, self.prefix)
+ signode += addnodes.desc_sig_space()
+ self.nestedName.describe_signature(signode, mode, env, symbol=symbol)
+ if self.placeholderType is not None:
+ signode += addnodes.desc_sig_space()
+ if self.placeholderType == 'auto':
+ signode += addnodes.desc_sig_keyword('auto', 'auto')
+ elif self.placeholderType == 'decltype(auto)':
+ signode += addnodes.desc_sig_keyword('decltype', 'decltype')
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ signode += addnodes.desc_sig_keyword('auto', 'auto')
+ signode += addnodes.desc_sig_punctuation(')', ')')
+ else:
+ raise AssertionError(self.placeholderType)
-class ASTFunctionParameter(ASTBase):
- def __init__(self, arg: (ASTTypeWithInit |
- ASTTemplateParamConstrainedTypeWithInit), ellipsis: bool=False) ->None:
+class ASTFunctionParameter(ASTBase):
+ def __init__(self, arg: ASTTypeWithInit | ASTTemplateParamConstrainedTypeWithInit,
+ ellipsis: bool = False) -> None:
self.arg = arg
self.ellipsis = ellipsis
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTFunctionParameter):
return NotImplemented
return self.arg == other.arg and self.ellipsis == other.ellipsis
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.arg, self.ellipsis))
+ def get_id(
+ self, version: int, objectType: str | None = None, symbol: Symbol | None = None,
+ ) -> str:
+ # this is not part of the normal name mangling in C++
+ if symbol:
+ # the anchor will be our parent
+ return symbol.parent.declaration.get_id(version, prefixed=False)
+ # else, do the usual
+ if self.ellipsis:
+ return 'z'
+ else:
+ return self.arg.get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.ellipsis:
+ return '...'
+ else:
+ return transform(self.arg)
-class ASTNoexceptSpec(ASTBase):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.ellipsis:
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ else:
+ self.arg.describe_signature(signode, mode, env, symbol=symbol)
- def __init__(self, expr: (ASTExpression | None)) ->None:
+
+class ASTNoexceptSpec(ASTBase):
+ def __init__(self, expr: ASTExpression | None) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNoexceptSpec):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.expr:
+ return 'noexcept(' + transform(self.expr) + ')'
+ return 'noexcept'
-class ASTParametersQualifiers(ASTBase):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('noexcept', 'noexcept')
+ if self.expr:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.expr.describe_signature(signode, 'markType', env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
- def __init__(self, args: list[ASTFunctionParameter], volatile: bool,
- const: bool, refQual: (str | None), exceptionSpec: ASTNoexceptSpec,
- trailingReturn: ASTType, override: bool, final: bool, attrs:
- ASTAttributeList, initializer: (str | None)) ->None:
+
+class ASTParametersQualifiers(ASTBase):
+ def __init__(self, args: list[ASTFunctionParameter], volatile: bool, const: bool,
+ refQual: str | None, exceptionSpec: ASTNoexceptSpec,
+ trailingReturn: ASTType,
+ override: bool, final: bool, attrs: ASTAttributeList,
+ initializer: str | None) -> None:
self.args = args
self.volatile = volatile
self.const = const
@@ -791,42 +1950,197 @@ class ASTParametersQualifiers(ASTBase):
self.attrs = attrs
self.initializer = initializer
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTParametersQualifiers):
return NotImplemented
- return (self.args == other.args and self.volatile == other.volatile and
- self.const == other.const and self.refQual == other.refQual and
- self.exceptionSpec == other.exceptionSpec and self.
- trailingReturn == other.trailingReturn and self.override ==
- other.override and self.final == other.final and self.attrs ==
- other.attrs and self.initializer == other.initializer)
-
- def __hash__(self) ->int:
- return hash((self.args, self.volatile, self.const, self.refQual,
- self.exceptionSpec, self.trailingReturn, self.override, self.
- final, self.attrs, self.initializer))
+ return (
+ self.args == other.args
+ and self.volatile == other.volatile
+ and self.const == other.const
+ and self.refQual == other.refQual
+ and self.exceptionSpec == other.exceptionSpec
+ and self.trailingReturn == other.trailingReturn
+ and self.override == other.override
+ and self.final == other.final
+ and self.attrs == other.attrs
+ and self.initializer == other.initializer
+ )
+
+ def __hash__(self) -> int:
+ return hash((
+ self.args, self.volatile, self.const, self.refQual, self.exceptionSpec,
+ self.trailingReturn, self.override, self.final, self.attrs, self.initializer
+ ))
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.args
+
+ def get_modifiers_id(self, version: int) -> str:
+ res = []
+ if self.volatile:
+ res.append('V')
+ if self.const:
+ if version == 1:
+ res.append('C')
+ else:
+ res.append('K')
+ if self.refQual == '&&':
+ res.append('O')
+ elif self.refQual == '&':
+ res.append('R')
+ return ''.join(res)
+
+ def get_param_id(self, version: int) -> str:
+ if version == 1:
+ if len(self.args) == 0:
+ return ''
+ else:
+ return '__' + '.'.join(a.get_id(version) for a in self.args)
+ if len(self.args) == 0:
+ return 'v'
+ else:
+ return ''.join(a.get_id(version) for a in self.args)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append('(')
+ first = True
+ for a in self.args:
+ if not first:
+ res.append(', ')
+ first = False
+ res.append(str(a))
+ res.append(')')
+ if self.volatile:
+ res.append(' volatile')
+ if self.const:
+ res.append(' const')
+ if self.refQual:
+ res.append(' ')
+ res.append(self.refQual)
+ if self.exceptionSpec:
+ res.append(' ')
+ res.append(transform(self.exceptionSpec))
+ if self.trailingReturn:
+ res.append(' -> ')
+ res.append(transform(self.trailingReturn))
+ if self.final:
+ res.append(' final')
+ if self.override:
+ res.append(' override')
+ if len(self.attrs) != 0:
+ res.append(' ')
+ res.append(transform(self.attrs))
+ if self.initializer:
+ res.append(' = ')
+ res.append(self.initializer)
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ multi_line_parameter_list = False
+ test_node: Element = signode
+ while test_node.parent:
+ if not isinstance(test_node, addnodes.desc_signature):
+ test_node = test_node.parent
+ continue
+ multi_line_parameter_list = test_node.get('multi_line_parameter_list', False)
+ break
+
+ # only use the desc_parameterlist for the outer list, not for inner lists
+ if mode == 'lastIsName':
+ paramlist = addnodes.desc_parameterlist()
+ paramlist['multi_line_parameter_list'] = multi_line_parameter_list
+ for arg in self.args:
+ param = addnodes.desc_parameter('', '', noemph=True)
+ arg.describe_signature(param, 'param', env, symbol=symbol)
+ paramlist += param
+ signode += paramlist
+ else:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ first = True
+ for arg in self.args:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ first = False
+ arg.describe_signature(signode, 'markType', env, symbol=symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+
+ def _add_anno(signode: TextElement, text: str) -> None:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_keyword(text, text)
+
+ if self.volatile:
+ _add_anno(signode, 'volatile')
+ if self.const:
+ _add_anno(signode, 'const')
+ if self.refQual:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation(self.refQual, self.refQual)
+ if self.exceptionSpec:
+ signode += addnodes.desc_sig_space()
+ self.exceptionSpec.describe_signature(signode, mode, env, symbol)
+ if self.trailingReturn:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_operator('->', '->')
+ signode += addnodes.desc_sig_space()
+ self.trailingReturn.describe_signature(signode, mode, env, symbol)
+ if self.final:
+ _add_anno(signode, 'final')
+ if self.override:
+ _add_anno(signode, 'override')
+ if len(self.attrs) != 0:
+ signode += addnodes.desc_sig_space()
+ self.attrs.describe_signature(signode)
+ if self.initializer:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation('=', '=')
+ signode += addnodes.desc_sig_space()
+ assert self.initializer in ('0', 'delete', 'default')
+ if self.initializer == '0':
+ signode += addnodes.desc_sig_literal_number('0', '0')
+ else:
+ signode += addnodes.desc_sig_keyword(self.initializer, self.initializer)
class ASTExplicitSpec(ASTBase):
-
- def __init__(self, expr: (ASTExpression | None)) ->None:
+ def __init__(self, expr: ASTExpression | None) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTExplicitSpec):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['explicit']
+ if self.expr is not None:
+ res.append('(')
+ res.append(transform(self.expr))
+ res.append(')')
+ return ''.join(res)
-class ASTDeclSpecsSimple(ASTBase):
+ def describe_signature(self, signode: TextElement,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('explicit', 'explicit')
+ if self.expr is not None:
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.expr.describe_signature(signode, 'markType', env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
- def __init__(self, storage: str, threadLocal: bool, inline: bool,
- virtual: bool, explicitSpec: (ASTExplicitSpec | None), consteval:
- bool, constexpr: bool, constinit: bool, volatile: bool, const: bool,
- friend: bool, attrs: ASTAttributeList) ->None:
+
+class ASTDeclSpecsSimple(ASTBase):
+ def __init__(self, storage: str, threadLocal: bool, inline: bool, virtual: bool,
+ explicitSpec: ASTExplicitSpec | None,
+ consteval: bool, constexpr: bool, constinit: bool,
+ volatile: bool, const: bool, friend: bool,
+ attrs: ASTAttributeList) -> None:
self.storage = storage
self.threadLocal = threadLocal
self.inline = inline
@@ -840,152 +2154,717 @@ class ASTDeclSpecsSimple(ASTBase):
self.friend = friend
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclSpecsSimple):
return NotImplemented
- return (self.storage == other.storage and self.threadLocal == other
- .threadLocal and self.inline == other.inline and self.virtual ==
- other.virtual and self.explicitSpec == other.explicitSpec and
- self.consteval == other.consteval and self.constexpr == other.
- constexpr and self.constinit == other.constinit and self.
- volatile == other.volatile and self.const == other.const and
- self.friend == other.friend and self.attrs == other.attrs)
-
- def __hash__(self) ->int:
- return hash((self.storage, self.threadLocal, self.inline, self.
- virtual, self.explicitSpec, self.consteval, self.constexpr,
- self.constinit, self.volatile, self.const, self.friend, self.attrs)
- )
+ return (
+ self.storage == other.storage
+ and self.threadLocal == other.threadLocal
+ and self.inline == other.inline
+ and self.virtual == other.virtual
+ and self.explicitSpec == other.explicitSpec
+ and self.consteval == other.consteval
+ and self.constexpr == other.constexpr
+ and self.constinit == other.constinit
+ and self.volatile == other.volatile
+ and self.const == other.const
+ and self.friend == other.friend
+ and self.attrs == other.attrs
+ )
+
+ def __hash__(self) -> int:
+ return hash((
+ self.storage,
+ self.threadLocal,
+ self.inline,
+ self.virtual,
+ self.explicitSpec,
+ self.consteval,
+ self.constexpr,
+ self.constinit,
+ self.volatile,
+ self.const,
+ self.friend,
+ self.attrs,
+ ))
+
+ def mergeWith(self, other: ASTDeclSpecsSimple) -> ASTDeclSpecsSimple:
+ if not other:
+ return self
+ return ASTDeclSpecsSimple(self.storage or other.storage,
+ self.threadLocal or other.threadLocal,
+ self.inline or other.inline,
+ self.virtual or other.virtual,
+ self.explicitSpec or other.explicitSpec,
+ self.consteval or other.consteval,
+ self.constexpr or other.constexpr,
+ self.constinit or other.constinit,
+ self.volatile or other.volatile,
+ self.const or other.const,
+ self.friend or other.friend,
+ self.attrs + other.attrs)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res: list[str] = []
+ if len(self.attrs) != 0:
+ res.append(transform(self.attrs))
+ if self.storage:
+ res.append(self.storage)
+ if self.threadLocal:
+ res.append('thread_local')
+ if self.inline:
+ res.append('inline')
+ if self.friend:
+ res.append('friend')
+ if self.virtual:
+ res.append('virtual')
+ if self.explicitSpec:
+ res.append(transform(self.explicitSpec))
+ if self.consteval:
+ res.append('consteval')
+ if self.constexpr:
+ res.append('constexpr')
+ if self.constinit:
+ res.append('constinit')
+ if self.volatile:
+ res.append('volatile')
+ if self.const:
+ res.append('const')
+ return ' '.join(res)
+
+ def describe_signature(self, signode: TextElement,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.attrs.describe_signature(signode)
+ addSpace = len(self.attrs) != 0
+
+ def _add(signode: TextElement, text: str) -> bool:
+ if addSpace:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_keyword(text, text)
+ return True
+
+ if self.storage:
+ addSpace = _add(signode, self.storage)
+ if self.threadLocal:
+ addSpace = _add(signode, 'thread_local')
+ if self.inline:
+ addSpace = _add(signode, 'inline')
+ if self.friend:
+ addSpace = _add(signode, 'friend')
+ if self.virtual:
+ addSpace = _add(signode, 'virtual')
+ if self.explicitSpec:
+ if addSpace:
+ signode += addnodes.desc_sig_space()
+ self.explicitSpec.describe_signature(signode, env, symbol)
+ addSpace = True
+ if self.consteval:
+ addSpace = _add(signode, 'consteval')
+ if self.constexpr:
+ addSpace = _add(signode, 'constexpr')
+ if self.constinit:
+ addSpace = _add(signode, 'constinit')
+ if self.volatile:
+ addSpace = _add(signode, 'volatile')
+ if self.const:
+ addSpace = _add(signode, 'const')
class ASTDeclSpecs(ASTBase):
-
- def __init__(self, outer: str, leftSpecs: ASTDeclSpecsSimple,
- rightSpecs: ASTDeclSpecsSimple, trailing: ASTTrailingTypeSpec) ->None:
+ def __init__(self, outer: str,
+ leftSpecs: ASTDeclSpecsSimple, rightSpecs: ASTDeclSpecsSimple,
+ trailing: ASTTrailingTypeSpec) -> None:
+ # leftSpecs and rightSpecs are used for output
+ # allSpecs are used for id generation
self.outer = outer
self.leftSpecs = leftSpecs
self.rightSpecs = rightSpecs
self.allSpecs = self.leftSpecs.mergeWith(self.rightSpecs)
self.trailingTypeSpec = trailing
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclSpecs):
return NotImplemented
- return (self.outer == other.outer and self.leftSpecs == other.
- leftSpecs and self.rightSpecs == other.rightSpecs and self.
- trailingTypeSpec == other.trailingTypeSpec)
-
- def __hash__(self) ->int:
- return hash((self.outer, self.leftSpecs, self.rightSpecs, self.
- trailingTypeSpec))
-
+ return (
+ self.outer == other.outer
+ and self.leftSpecs == other.leftSpecs
+ and self.rightSpecs == other.rightSpecs
+ and self.trailingTypeSpec == other.trailingTypeSpec
+ )
+
+ def __hash__(self) -> int:
+ return hash((
+ self.outer,
+ self.leftSpecs,
+ self.rightSpecs,
+ self.trailingTypeSpec,
+ ))
+
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ res = []
+ res.append(self.trailingTypeSpec.get_id(version))
+ if self.allSpecs.volatile:
+ res.append('V')
+ if self.allSpecs.const:
+ res.append('C')
+ return ''.join(res)
+ res = []
+ if self.allSpecs.volatile:
+ res.append('V')
+ if self.allSpecs.const:
+ res.append('K')
+ if self.trailingTypeSpec is not None:
+ res.append(self.trailingTypeSpec.get_id(version))
+ return ''.join(res)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res: list[str] = []
+ l = transform(self.leftSpecs)
+ if len(l) > 0:
+ res.append(l)
+ if self.trailingTypeSpec:
+ if len(res) > 0:
+ res.append(" ")
+ res.append(transform(self.trailingTypeSpec))
+ r = str(self.rightSpecs)
+ if len(r) > 0:
+ if len(res) > 0:
+ res.append(" ")
+ res.append(r)
+ return "".join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ numChildren = len(signode)
+ self.leftSpecs.describe_signature(signode, env, symbol)
+ addSpace = len(signode) != numChildren
+
+ if self.trailingTypeSpec:
+ if addSpace:
+ signode += addnodes.desc_sig_space()
+ numChildren = len(signode)
+ self.trailingTypeSpec.describe_signature(signode, mode, env,
+ symbol=symbol)
+ addSpace = len(signode) != numChildren
+
+ if len(str(self.rightSpecs)) > 0:
+ if addSpace:
+ signode += addnodes.desc_sig_space()
+ self.rightSpecs.describe_signature(signode, env, symbol)
+
+
+# Declarator
+################################################################################
class ASTArray(ASTBase):
-
- def __init__(self, size: ASTExpression) ->None:
+ def __init__(self, size: ASTExpression) -> None:
self.size = size
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTArray):
return NotImplemented
return self.size == other.size
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.size)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.size:
+ return '[' + transform(self.size) + ']'
+ else:
+ return '[]'
+
+ def get_id(self, version: int) -> str:
+ if version == 1:
+ return 'A'
+ if version == 2:
+ if self.size:
+ return 'A' + str(self.size) + '_'
+ else:
+ return 'A_'
+ if self.size:
+ return 'A' + self.size.get_id(version) + '_'
+ else:
+ return 'A_'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('[', '[')
+ if self.size:
+ self.size.describe_signature(signode, 'markType', env, symbol)
+ signode += addnodes.desc_sig_punctuation(']', ']')
+
class ASTDeclarator(ASTBase):
- pass
+ @property
+ def name(self) -> ASTNestedName:
+ raise NotImplementedError(repr(self))
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ raise NotImplementedError(repr(self))
-class ASTDeclaratorNameParamQual(ASTDeclarator):
+ @property
+ def isPack(self) -> bool:
+ raise NotImplementedError(repr(self))
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ raise NotImplementedError(repr(self))
+
+ @property
+ def trailingReturn(self) -> ASTType:
+ raise NotImplementedError(repr(self))
- def __init__(self, declId: ASTNestedName, arrayOps: list[ASTArray],
- paramQual: ASTParametersQualifiers) ->None:
+ def require_space_after_declSpecs(self) -> bool:
+ raise NotImplementedError(repr(self))
+
+ def get_modifiers_id(self, version: int) -> str:
+ raise NotImplementedError(repr(self))
+
+ def get_param_id(self, version: int) -> str:
+ raise NotImplementedError(repr(self))
+
+ def get_ptr_suffix_id(self, version: int) -> str:
+ raise NotImplementedError(repr(self))
+
+ def get_type_id(self, version: int, returnTypeId: str) -> str:
+ raise NotImplementedError(repr(self))
+
+ def is_function_type(self) -> bool:
+ raise NotImplementedError(repr(self))
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ raise NotImplementedError(repr(self))
+
+
+class ASTDeclaratorNameParamQual(ASTDeclarator):
+ def __init__(self, declId: ASTNestedName,
+ arrayOps: list[ASTArray],
+ paramQual: ASTParametersQualifiers) -> None:
self.declId = declId
self.arrayOps = arrayOps
self.paramQual = paramQual
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorNameParamQual):
return NotImplemented
- return (self.declId == other.declId and self.arrayOps == other.
- arrayOps and self.paramQual == other.paramQual)
+ return (
+ self.declId == other.declId
+ and self.arrayOps == other.arrayOps
+ and self.paramQual == other.paramQual
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.declId, self.arrayOps, self.paramQual))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.declId
-class ASTDeclaratorNameBitField(ASTDeclarator):
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ self.declId = name
+
+ @property
+ def isPack(self) -> bool:
+ return False
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.paramQual.function_params
+
+ @property
+ def trailingReturn(self) -> ASTType:
+ return self.paramQual.trailingReturn
- def __init__(self, declId: ASTNestedName, size: ASTExpression) ->None:
+ # only the modifiers for a function, e.g.,
+ def get_modifiers_id(self, version: int) -> str:
+ # cv-qualifiers
+ if self.paramQual:
+ return self.paramQual.get_modifiers_id(version)
+ raise Exception("This should only be called on a function: %s" % self)
+
+ def get_param_id(self, version: int) -> str: # only the parameters (if any)
+ if self.paramQual:
+ return self.paramQual.get_param_id(version)
+ else:
+ return ''
+
+ def get_ptr_suffix_id(self, version: int) -> str: # only the array specifiers
+ return ''.join(a.get_id(version) for a in self.arrayOps)
+
+ def get_type_id(self, version: int, returnTypeId: str) -> str:
+ assert version >= 2
+ res = []
+ # TODO: can we actually have both array ops and paramQual?
+ res.append(self.get_ptr_suffix_id(version))
+ if self.paramQual:
+ res.append(self.get_modifiers_id(version))
+ res.append('F')
+ res.append(returnTypeId)
+ res.append(self.get_param_id(version))
+ res.append('E')
+ else:
+ res.append(returnTypeId)
+ return ''.join(res)
+
+ # ------------------------------------------------------------------------
+
+ def require_space_after_declSpecs(self) -> bool:
+ return self.declId is not None
+
+ def is_function_type(self) -> bool:
+ return self.paramQual is not None
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.declId:
+ res.append(transform(self.declId))
+ res.extend(transform(op) for op in self.arrayOps)
+ if self.paramQual:
+ res.append(transform(self.paramQual))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.declId:
+ self.declId.describe_signature(signode, mode, env, symbol)
+ for op in self.arrayOps:
+ op.describe_signature(signode, mode, env, symbol)
+ if self.paramQual:
+ self.paramQual.describe_signature(signode, mode, env, symbol)
+
+
+class ASTDeclaratorNameBitField(ASTDeclarator):
+ def __init__(self, declId: ASTNestedName, size: ASTExpression) -> None:
self.declId = declId
self.size = size
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorNameBitField):
return NotImplemented
return self.declId == other.declId and self.size == other.size
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.declId, self.size))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.declId
-class ASTDeclaratorPtr(ASTDeclarator):
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ self.declId = name
+
+ def get_param_id(self, version: int) -> str: # only the parameters (if any)
+ return ''
+
+ def get_ptr_suffix_id(self, version: int) -> str: # only the array specifiers
+ return ''
+ # ------------------------------------------------------------------------
+
+ def require_space_after_declSpecs(self) -> bool:
+ return self.declId is not None
+
+ def is_function_type(self) -> bool:
+ return False
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.declId:
+ res.append(transform(self.declId))
+ res.append(" : ")
+ res.append(transform(self.size))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.declId:
+ self.declId.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation(':', ':')
+ signode += addnodes.desc_sig_space()
+ self.size.describe_signature(signode, mode, env, symbol)
+
+
+class ASTDeclaratorPtr(ASTDeclarator):
def __init__(self, next: ASTDeclarator, volatile: bool, const: bool,
- attrs: ASTAttributeList) ->None:
+ attrs: ASTAttributeList) -> None:
assert next
self.next = next
self.volatile = volatile
self.const = const
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorPtr):
return NotImplemented
- return (self.next == other.next and self.volatile == other.volatile and
- self.const == other.const and self.attrs == other.attrs)
-
- def __hash__(self) ->int:
+ return (
+ self.next == other.next
+ and self.volatile == other.volatile
+ and self.const == other.const
+ and self.attrs == other.attrs
+ )
+
+ def __hash__(self) -> int:
return hash((self.next, self.volatile, self.const, self.attrs))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.next.name
+
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ self.next.name = name
+
+ @property
+ def isPack(self) -> bool:
+ return self.next.isPack
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.next.function_params
+
+ @property
+ def trailingReturn(self) -> ASTType:
+ return self.next.trailingReturn
+
+ def require_space_after_declSpecs(self) -> bool:
+ return self.next.require_space_after_declSpecs()
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['*']
+ res.append(transform(self.attrs))
+ if len(self.attrs) != 0 and (self.volatile or self.const):
+ res.append(' ')
+ if self.volatile:
+ res.append('volatile')
+ if self.const:
+ if self.volatile:
+ res.append(' ')
+ res.append('const')
+ if self.const or self.volatile or len(self.attrs) > 0:
+ if self.next.require_space_after_declSpecs():
+ res.append(' ')
+ res.append(transform(self.next))
+ return ''.join(res)
+
+ def get_modifiers_id(self, version: int) -> str:
+ return self.next.get_modifiers_id(version)
+
+ def get_param_id(self, version: int) -> str:
+ return self.next.get_param_id(version)
+
+ def get_ptr_suffix_id(self, version: int) -> str:
+ if version == 1:
+ res = ['P']
+ if self.volatile:
+ res.append('V')
+ if self.const:
+ res.append('C')
+ res.append(self.next.get_ptr_suffix_id(version))
+ return ''.join(res)
+
+ res = [self.next.get_ptr_suffix_id(version)]
+ res.append('P')
+ if self.volatile:
+ res.append('V')
+ if self.const:
+ res.append('C')
+ return ''.join(res)
+
+ def get_type_id(self, version: int, returnTypeId: str) -> str:
+ # ReturnType *next, so we are part of the return type of 'next
+ res = ['P']
+ if self.volatile:
+ res.append('V')
+ if self.const:
+ res.append('C')
+ res.append(returnTypeId)
+ return self.next.get_type_id(version, returnTypeId=''.join(res))
+
+ def is_function_type(self) -> bool:
+ return self.next.is_function_type()
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('*', '*')
+ self.attrs.describe_signature(signode)
+ if len(self.attrs) != 0 and (self.volatile or self.const):
+ signode += addnodes.desc_sig_space()
+
+ def _add_anno(signode: TextElement, text: str) -> None:
+ signode += addnodes.desc_sig_keyword(text, text)
+ if self.volatile:
+ _add_anno(signode, 'volatile')
+ if self.const:
+ if self.volatile:
+ signode += addnodes.desc_sig_space()
+ _add_anno(signode, 'const')
+ if self.const or self.volatile or len(self.attrs) > 0:
+ if self.next.require_space_after_declSpecs():
+ signode += addnodes.desc_sig_space()
+ self.next.describe_signature(signode, mode, env, symbol)
-class ASTDeclaratorRef(ASTDeclarator):
- def __init__(self, next: ASTDeclarator, attrs: ASTAttributeList) ->None:
+class ASTDeclaratorRef(ASTDeclarator):
+ def __init__(self, next: ASTDeclarator, attrs: ASTAttributeList) -> None:
assert next
self.next = next
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorRef):
return NotImplemented
return self.next == other.next and self.attrs == other.attrs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.next, self.attrs))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.next.name
-class ASTDeclaratorParamPack(ASTDeclarator):
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ self.next.name = name
+
+ @property
+ def isPack(self) -> bool:
+ return self.next.isPack
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.next.function_params
- def __init__(self, next: ASTDeclarator) ->None:
+ @property
+ def trailingReturn(self) -> ASTType:
+ return self.next.trailingReturn
+
+ def require_space_after_declSpecs(self) -> bool:
+ return self.next.require_space_after_declSpecs()
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['&']
+ res.append(transform(self.attrs))
+ if len(self.attrs) != 0 and self.next.require_space_after_declSpecs():
+ res.append(' ')
+ res.append(transform(self.next))
+ return ''.join(res)
+
+ def get_modifiers_id(self, version: int) -> str:
+ return self.next.get_modifiers_id(version)
+
+ def get_param_id(self, version: int) -> str: # only the parameters (if any)
+ return self.next.get_param_id(version)
+
+ def get_ptr_suffix_id(self, version: int) -> str:
+ if version == 1:
+ return 'R' + self.next.get_ptr_suffix_id(version)
+ else:
+ return self.next.get_ptr_suffix_id(version) + 'R'
+
+ def get_type_id(self, version: int, returnTypeId: str) -> str:
+ assert version >= 2
+ # ReturnType &next, so we are part of the return type of 'next
+ return self.next.get_type_id(version, returnTypeId='R' + returnTypeId)
+
+ def is_function_type(self) -> bool:
+ return self.next.is_function_type()
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('&', '&')
+ self.attrs.describe_signature(signode)
+ if len(self.attrs) > 0 and self.next.require_space_after_declSpecs():
+ signode += addnodes.desc_sig_space()
+ self.next.describe_signature(signode, mode, env, symbol)
+
+
+class ASTDeclaratorParamPack(ASTDeclarator):
+ def __init__(self, next: ASTDeclarator) -> None:
assert next
self.next = next
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorParamPack):
return NotImplemented
return self.next == other.next
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.next)
+ @property
+ def name(self) -> ASTNestedName:
+ return self.next.name
-class ASTDeclaratorMemPtr(ASTDeclarator):
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ self.next.name = name
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.next.function_params
- def __init__(self, className: ASTNestedName, const: bool, volatile:
- bool, next: ASTDeclarator) ->None:
+ @property
+ def trailingReturn(self) -> ASTType:
+ return self.next.trailingReturn
+
+ @property
+ def isPack(self) -> bool:
+ return True
+
+ def require_space_after_declSpecs(self) -> bool:
+ return False
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = transform(self.next)
+ if self.next.name:
+ res = ' ' + res
+ return '...' + res
+
+ def get_modifiers_id(self, version: int) -> str:
+ return self.next.get_modifiers_id(version)
+
+ def get_param_id(self, version: int) -> str: # only the parameters (if any)
+ return self.next.get_param_id(version)
+
+ def get_ptr_suffix_id(self, version: int) -> str:
+ if version == 1:
+ return 'Dp' + self.next.get_ptr_suffix_id(version)
+ else:
+ return self.next.get_ptr_suffix_id(version) + 'Dp'
+
+ def get_type_id(self, version: int, returnTypeId: str) -> str:
+ assert version >= 2
+ # ReturnType... next, so we are part of the return type of 'next
+ return self.next.get_type_id(version, returnTypeId='Dp' + returnTypeId)
+
+ def is_function_type(self) -> bool:
+ return self.next.is_function_type()
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ if self.next.name:
+ signode += addnodes.desc_sig_space()
+ self.next.describe_signature(signode, mode, env, symbol)
+
+
+class ASTDeclaratorMemPtr(ASTDeclarator):
+ def __init__(self, className: ASTNestedName,
+ const: bool, volatile: bool, next: ASTDeclarator) -> None:
assert className
assert next
self.className = className
@@ -993,258 +2872,851 @@ class ASTDeclaratorMemPtr(ASTDeclarator):
self.volatile = volatile
self.next = next
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorMemPtr):
return NotImplemented
- return (self.className == other.className and self.const == other.
- const and self.volatile == other.volatile and self.next ==
- other.next)
-
- def __hash__(self) ->int:
+ return (
+ self.className == other.className
+ and self.const == other.const
+ and self.volatile == other.volatile
+ and self.next == other.next
+ )
+
+ def __hash__(self) -> int:
return hash((self.className, self.const, self.volatile, self.next))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.next.name
+
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ self.next.name = name
+
+ @property
+ def isPack(self) -> bool:
+ return self.next.isPack
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.next.function_params
+
+ @property
+ def trailingReturn(self) -> ASTType:
+ return self.next.trailingReturn
+
+ def require_space_after_declSpecs(self) -> bool:
+ return True
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.className))
+ res.append('::*')
+ if self.volatile:
+ res.append('volatile')
+ if self.const:
+ if self.volatile:
+ res.append(' ')
+ res.append('const')
+ if self.next.require_space_after_declSpecs():
+ res.append(' ')
+ res.append(transform(self.next))
+ return ''.join(res)
+
+ def get_modifiers_id(self, version: int) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return self.next.get_modifiers_id(version)
+
+ def get_param_id(self, version: int) -> str: # only the parameters (if any)
+ if version == 1:
+ raise NoOldIdError
+ return self.next.get_param_id(version)
+
+ def get_ptr_suffix_id(self, version: int) -> str:
+ if version == 1:
+ raise NoOldIdError
+ raise NotImplementedError
+ return self.next.get_ptr_suffix_id(version) + 'Dp'
+
+ def get_type_id(self, version: int, returnTypeId: str) -> str:
+ assert version >= 2
+ # ReturnType name::* next, so we are part of the return type of next
+ nextReturnTypeId = ''
+ if self.volatile:
+ nextReturnTypeId += 'V'
+ if self.const:
+ nextReturnTypeId += 'K'
+ nextReturnTypeId += 'M'
+ nextReturnTypeId += self.className.get_id(version)
+ nextReturnTypeId += returnTypeId
+ return self.next.get_type_id(version, nextReturnTypeId)
+
+ def is_function_type(self) -> bool:
+ return self.next.is_function_type()
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.className.describe_signature(signode, 'markType', env, symbol)
+ signode += addnodes.desc_sig_punctuation('::', '::')
+ signode += addnodes.desc_sig_punctuation('*', '*')
+
+ def _add_anno(signode: TextElement, text: str) -> None:
+ signode += addnodes.desc_sig_keyword(text, text)
+ if self.volatile:
+ _add_anno(signode, 'volatile')
+ if self.const:
+ if self.volatile:
+ signode += addnodes.desc_sig_space()
+ _add_anno(signode, 'const')
+ if self.next.require_space_after_declSpecs():
+ signode += addnodes.desc_sig_space()
+ self.next.describe_signature(signode, mode, env, symbol)
-class ASTDeclaratorParen(ASTDeclarator):
- def __init__(self, inner: ASTDeclarator, next: ASTDeclarator) ->None:
+class ASTDeclaratorParen(ASTDeclarator):
+ def __init__(self, inner: ASTDeclarator, next: ASTDeclarator) -> None:
assert inner
assert next
self.inner = inner
self.next = next
+ # TODO: we assume the name, params, and qualifiers are in inner
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaratorParen):
return NotImplemented
return self.inner == other.inner and self.next == other.next
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.inner, self.next))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.inner.name
-class ASTPackExpansionExpr(ASTExpression):
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ self.inner.name = name
+
+ @property
+ def isPack(self) -> bool:
+ return self.inner.isPack or self.next.isPack
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.inner.function_params
+
+ @property
+ def trailingReturn(self) -> ASTType:
+ return self.inner.trailingReturn
+
+ def require_space_after_declSpecs(self) -> bool:
+ return True
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = ['(']
+ res.append(transform(self.inner))
+ res.append(')')
+ res.append(transform(self.next))
+ return ''.join(res)
+
+ def get_modifiers_id(self, version: int) -> str:
+ return self.inner.get_modifiers_id(version)
+
+ def get_param_id(self, version: int) -> str: # only the parameters (if any)
+ return self.inner.get_param_id(version)
+
+ def get_ptr_suffix_id(self, version: int) -> str:
+ if version == 1:
+ raise NoOldIdError # TODO: was this implemented before?
+ return self.next.get_ptr_suffix_id(version) + \
+ self.inner.get_ptr_suffix_id(version)
+ return self.inner.get_ptr_suffix_id(version) + \
+ self.next.get_ptr_suffix_id(version)
+
+ def get_type_id(self, version: int, returnTypeId: str) -> str:
+ assert version >= 2
+ # ReturnType (inner)next, so 'inner' returns everything outside
+ nextId = self.next.get_type_id(version, returnTypeId)
+ return self.inner.get_type_id(version, returnTypeId=nextId)
+
+ def is_function_type(self) -> bool:
+ return self.inner.is_function_type()
- def __init__(self, expr: (ASTExpression | ASTBracedInitList)) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ self.inner.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
+ self.next.describe_signature(signode, "noneIsName", env, symbol)
+
+
+# Type and initializer stuff
+##############################################################################################
+
+class ASTPackExpansionExpr(ASTExpression):
+ def __init__(self, expr: ASTExpression | ASTBracedInitList) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTPackExpansionExpr):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.expr) + '...'
+
+ def get_id(self, version: int) -> str:
+ id = self.expr.get_id(version)
+ return 'sp' + id
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.expr.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation('...', '...')
-class ASTParenExprList(ASTBaseParenExprList):
- def __init__(self, exprs: list[ASTExpression | ASTBracedInitList]) ->None:
+class ASTParenExprList(ASTBaseParenExprList):
+ def __init__(self, exprs: list[ASTExpression | ASTBracedInitList]) -> None:
self.exprs = exprs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTParenExprList):
return NotImplemented
return self.exprs == other.exprs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.exprs)
+ def get_id(self, version: int) -> str:
+ return "pi%sE" % ''.join(e.get_id(version) for e in self.exprs)
-class ASTInitializer(ASTBase):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ exprs = [transform(e) for e in self.exprs]
+ return '(%s)' % ', '.join(exprs)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ signode += addnodes.desc_sig_punctuation('(', '(')
+ first = True
+ for e in self.exprs:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ else:
+ first = False
+ e.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation(')', ')')
- def __init__(self, value: (ASTExpression | ASTBracedInitList),
- hasAssign: bool=True) ->None:
+
+class ASTInitializer(ASTBase):
+ def __init__(self, value: ASTExpression | ASTBracedInitList,
+ hasAssign: bool = True) -> None:
self.value = value
self.hasAssign = hasAssign
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTInitializer):
return NotImplemented
return self.value == other.value and self.hasAssign == other.hasAssign
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.value, self.hasAssign))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ val = transform(self.value)
+ if self.hasAssign:
+ return ' = ' + val
+ else:
+ return val
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.hasAssign:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation('=', '=')
+ signode += addnodes.desc_sig_space()
+ self.value.describe_signature(signode, 'markType', env, symbol)
-class ASTType(ASTBase):
- def __init__(self, declSpecs: ASTDeclSpecs, decl: ASTDeclarator) ->None:
+class ASTType(ASTBase):
+ def __init__(self, declSpecs: ASTDeclSpecs, decl: ASTDeclarator) -> None:
assert declSpecs
assert decl
self.declSpecs = declSpecs
self.decl = decl
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTType):
return NotImplemented
return self.declSpecs == other.declSpecs and self.decl == other.decl
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.declSpecs, self.decl))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.decl.name
+
+ @name.setter
+ def name(self, name: ASTNestedName) -> None:
+ self.decl.name = name
+
+ @property
+ def isPack(self) -> bool:
+ return self.decl.isPack
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ return self.decl.function_params
+
+ @property
+ def trailingReturn(self) -> ASTType:
+ return self.decl.trailingReturn
+
+ def get_id(self, version: int, objectType: str | None = None,
+ symbol: Symbol | None = None) -> str:
+ if version == 1:
+ res = []
+ if objectType: # needs the name
+ if objectType == 'function': # also modifiers
+ res.append(symbol.get_full_nested_name().get_id(version))
+ res.append(self.decl.get_param_id(version))
+ res.append(self.decl.get_modifiers_id(version))
+ if (self.declSpecs.leftSpecs.constexpr or
+ (self.declSpecs.rightSpecs and
+ self.declSpecs.rightSpecs.constexpr)):
+ res.append('CE')
+ elif objectType == 'type': # just the name
+ res.append(symbol.get_full_nested_name().get_id(version))
+ else:
+ raise AssertionError(objectType)
+ else: # only type encoding
+ if self.decl.is_function_type():
+ raise NoOldIdError
+ res.append(self.declSpecs.get_id(version))
+ res.append(self.decl.get_ptr_suffix_id(version))
+ res.append(self.decl.get_param_id(version))
+ return ''.join(res)
+ # other versions
+ res = []
+ if objectType: # needs the name
+ if objectType == 'function': # also modifiers
+ modifiers = self.decl.get_modifiers_id(version)
+ res.append(symbol.get_full_nested_name().get_id(version, modifiers))
+ if version >= 4:
+ # with templates we need to mangle the return type in as well
+ templ = symbol.declaration.templatePrefix
+ if templ is not None:
+ typeId = self.decl.get_ptr_suffix_id(version)
+ if self.trailingReturn:
+ returnTypeId = self.trailingReturn.get_id(version)
+ else:
+ returnTypeId = self.declSpecs.get_id(version)
+ res.append(typeId)
+ res.append(returnTypeId)
+ res.append(self.decl.get_param_id(version))
+ elif objectType == 'type': # just the name
+ res.append(symbol.get_full_nested_name().get_id(version))
+ else:
+ raise AssertionError(objectType)
+ else: # only type encoding
+ # the 'returnType' of a non-function type is simply just the last
+ # type, i.e., for 'int*' it is 'int'
+ returnTypeId = self.declSpecs.get_id(version)
+ typeId = self.decl.get_type_id(version, returnTypeId)
+ res.append(typeId)
+ return ''.join(res)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ declSpecs = transform(self.declSpecs)
+ res.append(declSpecs)
+ if self.decl.require_space_after_declSpecs() and len(declSpecs) > 0:
+ res.append(' ')
+ res.append(transform(self.decl))
+ return ''.join(res)
+
+ def get_type_declaration_prefix(self) -> str:
+ if self.declSpecs.trailingTypeSpec:
+ return 'typedef'
+ else:
+ return 'type'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.declSpecs.describe_signature(signode, 'markType', env, symbol)
+ if (self.decl.require_space_after_declSpecs() and
+ len(str(self.declSpecs)) > 0):
+ signode += addnodes.desc_sig_space()
+ # for parameters that don't really declare new names we get 'markType',
+ # this should not be propagated, but be 'noneIsName'.
+ if mode == 'markType':
+ mode = 'noneIsName'
+ self.decl.describe_signature(signode, mode, env, symbol)
-class ASTTemplateParamConstrainedTypeWithInit(ASTBase):
- def __init__(self, type: ASTType, init: ASTType) ->None:
+class ASTTemplateParamConstrainedTypeWithInit(ASTBase):
+ def __init__(self, type: ASTType, init: ASTType) -> None:
assert type
self.type = type
self.init = init
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateParamConstrainedTypeWithInit):
return NotImplemented
return self.type == other.type and self.init == other.init
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.type, self.init))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.type.name
+
+ @property
+ def isPack(self) -> bool:
+ return self.type.isPack
+
+ def get_id(
+ self, version: int, objectType: str | None = None, symbol: Symbol | None = None,
+ ) -> str:
+ # this is not part of the normal name mangling in C++
+ assert version >= 2
+ if symbol:
+ # the anchor will be our parent
+ return symbol.parent.declaration.get_id(version, prefixed=False)
+ else:
+ return self.type.get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = transform(self.type)
+ if self.init:
+ res += " = "
+ res += transform(self.init)
+ return res
-class ASTTypeWithInit(ASTBase):
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.type.describe_signature(signode, mode, env, symbol)
+ if self.init:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation('=', '=')
+ signode += addnodes.desc_sig_space()
+ self.init.describe_signature(signode, mode, env, symbol)
- def __init__(self, type: ASTType, init: ASTInitializer) ->None:
+
+class ASTTypeWithInit(ASTBase):
+ def __init__(self, type: ASTType, init: ASTInitializer) -> None:
self.type = type
self.init = init
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTypeWithInit):
return NotImplemented
return self.type == other.type and self.init == other.init
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.type, self.init))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.type.name
+
+ @property
+ def isPack(self) -> bool:
+ return self.type.isPack
+
+ def get_id(self, version: int, objectType: str | None = None,
+ symbol: Symbol | None = None) -> str:
+ if objectType != 'member':
+ return self.type.get_id(version, objectType)
+ if version == 1:
+ return (symbol.get_full_nested_name().get_id(version) + '__' +
+ self.type.get_id(version))
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.type))
+ if self.init:
+ res.append(transform(self.init))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.type.describe_signature(signode, mode, env, symbol)
+ if self.init:
+ self.init.describe_signature(signode, mode, env, symbol)
-class ASTTypeUsing(ASTBase):
- def __init__(self, name: ASTNestedName, type: (ASTType | None)) ->None:
+class ASTTypeUsing(ASTBase):
+ def __init__(self, name: ASTNestedName, type: ASTType | None) -> None:
self.name = name
self.type = type
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTypeUsing):
return NotImplemented
return self.name == other.name and self.type == other.type
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.name, self.type))
+ def get_id(self, version: int, objectType: str | None = None,
+ symbol: Symbol | None = None) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.name))
+ if self.type:
+ res.append(' = ')
+ res.append(transform(self.type))
+ return ''.join(res)
+
+ def get_type_declaration_prefix(self) -> str:
+ return 'using'
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.name.describe_signature(signode, mode, env, symbol=symbol)
+ if self.type:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation('=', '=')
+ signode += addnodes.desc_sig_space()
+ self.type.describe_signature(signode, 'markType', env, symbol=symbol)
+
+
+# Other declarations
+##############################################################################################
class ASTConcept(ASTBase):
-
- def __init__(self, nestedName: ASTNestedName, initializer: ASTInitializer
- ) ->None:
+ def __init__(self, nestedName: ASTNestedName, initializer: ASTInitializer) -> None:
self.nestedName = nestedName
self.initializer = initializer
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTConcept):
return NotImplemented
- return (self.nestedName == other.nestedName and self.initializer ==
- other.initializer)
+ return self.nestedName == other.nestedName and self.initializer == other.initializer
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.nestedName, self.initializer))
+ @property
+ def name(self) -> ASTNestedName:
+ return self.nestedName
+
+ def get_id(self, version: int, objectType: str | None = None,
+ symbol: Symbol | None = None) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = transform(self.nestedName)
+ if self.initializer:
+ res += transform(self.initializer)
+ return res
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.nestedName.describe_signature(signode, mode, env, symbol)
+ if self.initializer:
+ self.initializer.describe_signature(signode, mode, env, symbol)
-class ASTBaseClass(ASTBase):
- def __init__(self, name: ASTNestedName, visibility: str, virtual: bool,
- pack: bool) ->None:
+class ASTBaseClass(ASTBase):
+ def __init__(self, name: ASTNestedName, visibility: str,
+ virtual: bool, pack: bool) -> None:
self.name = name
self.visibility = visibility
self.virtual = virtual
self.pack = pack
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTBaseClass):
return NotImplemented
- return (self.name == other.name and self.visibility == other.
- visibility and self.virtual == other.virtual and self.pack ==
- other.pack)
-
- def __hash__(self) ->int:
+ return (
+ self.name == other.name
+ and self.visibility == other.visibility
+ and self.virtual == other.virtual
+ and self.pack == other.pack
+ )
+
+ def __hash__(self) -> int:
return hash((self.name, self.visibility, self.virtual, self.pack))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.visibility is not None:
+ res.append(self.visibility)
+ res.append(' ')
+ if self.virtual:
+ res.append('virtual ')
+ res.append(transform(self.name))
+ if self.pack:
+ res.append('...')
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ if self.visibility is not None:
+ signode += addnodes.desc_sig_keyword(self.visibility,
+ self.visibility)
+ signode += addnodes.desc_sig_space()
+ if self.virtual:
+ signode += addnodes.desc_sig_keyword('virtual', 'virtual')
+ signode += addnodes.desc_sig_space()
+ self.name.describe_signature(signode, 'markType', env, symbol=symbol)
+ if self.pack:
+ signode += addnodes.desc_sig_punctuation('...', '...')
-class ASTClass(ASTBase):
- def __init__(self, name: ASTNestedName, final: bool, bases: list[
- ASTBaseClass], attrs: ASTAttributeList) ->None:
+class ASTClass(ASTBase):
+ def __init__(self, name: ASTNestedName, final: bool, bases: list[ASTBaseClass],
+ attrs: ASTAttributeList) -> None:
self.name = name
self.final = final
self.bases = bases
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTClass):
return NotImplemented
- return (self.name == other.name and self.final == other.final and
- self.bases == other.bases and self.attrs == other.attrs)
-
- def __hash__(self) ->int:
+ return (
+ self.name == other.name
+ and self.final == other.final
+ and self.bases == other.bases
+ and self.attrs == other.attrs
+ )
+
+ def __hash__(self) -> int:
return hash((self.name, self.final, self.bases, self.attrs))
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.attrs))
+ if len(self.attrs) != 0:
+ res.append(' ')
+ res.append(transform(self.name))
+ if self.final:
+ res.append(' final')
+ if len(self.bases) > 0:
+ res.append(' : ')
+ first = True
+ for b in self.bases:
+ if not first:
+ res.append(', ')
+ first = False
+ res.append(transform(b))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.attrs.describe_signature(signode)
+ if len(self.attrs) != 0:
+ signode += addnodes.desc_sig_space()
+ self.name.describe_signature(signode, mode, env, symbol=symbol)
+ if self.final:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_keyword('final', 'final')
+ if len(self.bases) > 0:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation(':', ':')
+ signode += addnodes.desc_sig_space()
+ for b in self.bases:
+ b.describe_signature(signode, mode, env, symbol=symbol)
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ signode.pop()
+ signode.pop()
-class ASTUnion(ASTBase):
- def __init__(self, name: ASTNestedName, attrs: ASTAttributeList) ->None:
+class ASTUnion(ASTBase):
+ def __init__(self, name: ASTNestedName, attrs: ASTAttributeList) -> None:
self.name = name
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTUnion):
return NotImplemented
return self.name == other.name and self.attrs == other.attrs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.name, self.attrs))
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return symbol.get_full_nested_name().get_id(version)
-class ASTEnum(ASTBase):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.attrs))
+ if len(self.attrs) != 0:
+ res.append(' ')
+ res.append(transform(self.name))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.attrs.describe_signature(signode)
+ if len(self.attrs) != 0:
+ signode += addnodes.desc_sig_space()
+ self.name.describe_signature(signode, mode, env, symbol=symbol)
- def __init__(self, name: ASTNestedName, scoped: str, underlyingType:
- ASTType, attrs: ASTAttributeList) ->None:
+
+class ASTEnum(ASTBase):
+ def __init__(self, name: ASTNestedName, scoped: str, underlyingType: ASTType,
+ attrs: ASTAttributeList) -> None:
self.name = name
self.scoped = scoped
self.underlyingType = underlyingType
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTEnum):
return NotImplemented
- return (self.name == other.name and self.scoped == other.scoped and
- self.underlyingType == other.underlyingType and self.attrs ==
- other.attrs)
-
- def __hash__(self) ->int:
+ return (
+ self.name == other.name
+ and self.scoped == other.scoped
+ and self.underlyingType == other.underlyingType
+ and self.attrs == other.attrs
+ )
+
+ def __hash__(self) -> int:
return hash((self.name, self.scoped, self.underlyingType, self.attrs))
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.scoped:
+ res.append(self.scoped)
+ res.append(' ')
+ res.append(transform(self.attrs))
+ if len(self.attrs) != 0:
+ res.append(' ')
+ res.append(transform(self.name))
+ if self.underlyingType:
+ res.append(' : ')
+ res.append(transform(self.underlyingType))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ # self.scoped has been done by the CPPEnumObject
+ self.attrs.describe_signature(signode)
+ if len(self.attrs) != 0:
+ signode += addnodes.desc_sig_space()
+ self.name.describe_signature(signode, mode, env, symbol=symbol)
+ if self.underlyingType:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation(':', ':')
+ signode += addnodes.desc_sig_space()
+ self.underlyingType.describe_signature(signode, 'noneIsName',
+ env, symbol=symbol)
-class ASTEnumerator(ASTBase):
- def __init__(self, name: ASTNestedName, init: (ASTInitializer | None),
- attrs: ASTAttributeList) ->None:
+class ASTEnumerator(ASTBase):
+ def __init__(self, name: ASTNestedName, init: ASTInitializer | None,
+ attrs: ASTAttributeList) -> None:
self.name = name
self.init = init
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTEnumerator):
return NotImplemented
- return (self.name == other.name and self.init == other.init and
- self.attrs == other.attrs)
+ return (
+ self.name == other.name
+ and self.init == other.init
+ and self.attrs == other.attrs
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.name, self.init, self.attrs))
+ def get_id(self, version: int, objectType: str, symbol: Symbol) -> str:
+ if version == 1:
+ raise NoOldIdError
+ return symbol.get_full_nested_name().get_id(version)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.name))
+ if len(self.attrs) != 0:
+ res.append(' ')
+ res.append(transform(self.attrs))
+ if self.init:
+ res.append(transform(self.init))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ verify_description_mode(mode)
+ self.name.describe_signature(signode, mode, env, symbol)
+ if len(self.attrs) != 0:
+ signode += addnodes.desc_sig_space()
+ self.attrs.describe_signature(signode)
+ if self.init:
+ self.init.describe_signature(signode, 'markType', env, symbol)
+
+
+################################################################################
+# Templates
+################################################################################
+
+# Parameters
+################################################################################
class ASTTemplateParam(ASTBase):
- pass
+ def get_identifier(self) -> ASTIdentifier:
+ raise NotImplementedError(repr(self))
+ def get_id(self, version: int) -> str:
+ raise NotImplementedError(repr(self))
-class ASTTemplateKeyParamPackIdDefault(ASTTemplateParam):
+ def describe_signature(self, parentNode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ raise NotImplementedError(repr(self))
+
+ @property
+ def isPack(self) -> bool:
+ raise NotImplementedError(repr(self))
+
+ @property
+ def name(self) -> ASTNestedName:
+ raise NotImplementedError(repr(self))
- def __init__(self, key: str, identifier: ASTIdentifier, parameterPack:
- bool, default: ASTType) ->None:
+
+class ASTTemplateKeyParamPackIdDefault(ASTTemplateParam):
+ def __init__(self, key: str, identifier: ASTIdentifier,
+ parameterPack: bool, default: ASTType) -> None:
assert key
if parameterPack:
assert default is None
@@ -1253,155 +3725,510 @@ class ASTTemplateKeyParamPackIdDefault(ASTTemplateParam):
self.parameterPack = parameterPack
self.default = default
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateKeyParamPackIdDefault):
return NotImplemented
- return (self.key == other.key and self.identifier == other.
- identifier and self.parameterPack == other.parameterPack and
- self.default == other.default)
-
- def __hash__(self) ->int:
- return hash((self.key, self.identifier, self.parameterPack, self.
- default))
+ return (
+ self.key == other.key
+ and self.identifier == other.identifier
+ and self.parameterPack == other.parameterPack
+ and self.default == other.default
+ )
+
+ def __hash__(self) -> int:
+ return hash((self.key, self.identifier, self.parameterPack, self.default))
+
+ def get_identifier(self) -> ASTIdentifier:
+ return self.identifier
+
+ def get_id(self, version: int) -> str:
+ assert version >= 2
+ # this is not part of the normal name mangling in C++
+ res = []
+ if self.parameterPack:
+ res.append('Dp')
+ else:
+ res.append('0') # we need to put something
+ return ''.join(res)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = [self.key]
+ if self.parameterPack:
+ if self.identifier:
+ res.append(' ')
+ res.append('...')
+ if self.identifier:
+ if not self.parameterPack:
+ res.append(' ')
+ res.append(transform(self.identifier))
+ if self.default:
+ res.append(' = ')
+ res.append(transform(self.default))
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword(self.key, self.key)
+ if self.parameterPack:
+ if self.identifier:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ if self.identifier:
+ if not self.parameterPack:
+ signode += addnodes.desc_sig_space()
+ self.identifier.describe_signature(signode, mode, env, '', '', symbol)
+ if self.default:
+ signode += addnodes.desc_sig_space()
+ signode += addnodes.desc_sig_punctuation('=', '=')
+ signode += addnodes.desc_sig_space()
+ self.default.describe_signature(signode, 'markType', env, symbol)
class ASTTemplateParamType(ASTTemplateParam):
-
- def __init__(self, data: ASTTemplateKeyParamPackIdDefault) ->None:
+ def __init__(self, data: ASTTemplateKeyParamPackIdDefault) -> None:
assert data
self.data = data
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateParamType):
return NotImplemented
return self.data == other.data
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.data)
+ @property
+ def name(self) -> ASTNestedName:
+ id = self.get_identifier()
+ return ASTNestedName([ASTNestedNameElement(id, None)], [False], rooted=False)
+
+ @property
+ def isPack(self) -> bool:
+ return self.data.parameterPack
+
+ def get_identifier(self) -> ASTIdentifier:
+ return self.data.get_identifier()
+
+ def get_id(
+ self, version: int, objectType: str | None = None, symbol: Symbol | None = None,
+ ) -> str:
+ # this is not part of the normal name mangling in C++
+ assert version >= 2
+ if symbol:
+ # the anchor will be our parent
+ return symbol.parent.declaration.get_id(version, prefixed=False)
+ else:
+ return self.data.get_id(version)
-class ASTTemplateParamTemplateType(ASTTemplateParam):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.data)
- def __init__(self, nestedParams: ASTTemplateParams, data:
- ASTTemplateKeyParamPackIdDefault) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.data.describe_signature(signode, mode, env, symbol)
+
+
+class ASTTemplateParamTemplateType(ASTTemplateParam):
+ def __init__(self, nestedParams: ASTTemplateParams,
+ data: ASTTemplateKeyParamPackIdDefault) -> None:
assert nestedParams
assert data
self.nestedParams = nestedParams
self.data = data
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateParamTemplateType):
return NotImplemented
- return (self.nestedParams == other.nestedParams and self.data ==
- other.data)
+ return (
+ self.nestedParams == other.nestedParams
+ and self.data == other.data
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.nestedParams, self.data))
+ @property
+ def name(self) -> ASTNestedName:
+ id = self.get_identifier()
+ return ASTNestedName([ASTNestedNameElement(id, None)], [False], rooted=False)
+
+ @property
+ def isPack(self) -> bool:
+ return self.data.parameterPack
+
+ def get_identifier(self) -> ASTIdentifier:
+ return self.data.get_identifier()
+
+ def get_id(
+ self, version: int, objectType: str | None = None, symbol: Symbol | None = None,
+ ) -> str:
+ assert version >= 2
+ # this is not part of the normal name mangling in C++
+ if symbol:
+ # the anchor will be our parent
+ return symbol.parent.declaration.get_id(version, prefixed=None)
+ else:
+ return self.nestedParams.get_id(version) + self.data.get_id(version)
-class ASTTemplateParamNonType(ASTTemplateParam):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return transform(self.nestedParams) + transform(self.data)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.nestedParams.describe_signature(signode, 'noneIsName', env, symbol)
+ signode += addnodes.desc_sig_space()
+ self.data.describe_signature(signode, mode, env, symbol)
- def __init__(self, param: (ASTTypeWithInit |
- ASTTemplateParamConstrainedTypeWithInit), parameterPack: bool=False
- ) ->None:
+
+class ASTTemplateParamNonType(ASTTemplateParam):
+ def __init__(self,
+ param: ASTTypeWithInit | ASTTemplateParamConstrainedTypeWithInit,
+ parameterPack: bool = False) -> None:
assert param
self.param = param
self.parameterPack = parameterPack
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateParamNonType):
return NotImplemented
- return (self.param == other.param and self.parameterPack == other.
- parameterPack)
+ return (
+ self.param == other.param
+ and self.parameterPack == other.parameterPack
+ )
+
+ @property
+ def name(self) -> ASTNestedName:
+ id = self.get_identifier()
+ return ASTNestedName([ASTNestedNameElement(id, None)], [False], rooted=False)
+
+ @property
+ def isPack(self) -> bool:
+ return self.param.isPack or self.parameterPack
+
+ def get_identifier(self) -> ASTIdentifier:
+ name = self.param.name
+ if name:
+ assert len(name.names) == 1
+ assert name.names[0].identOrOp
+ assert not name.names[0].templateArgs
+ res = name.names[0].identOrOp
+ assert isinstance(res, ASTIdentifier)
+ return res
+ else:
+ return None
+
+ def get_id(
+ self, version: int, objectType: str | None = None, symbol: Symbol | None = None,
+ ) -> str:
+ assert version >= 2
+ # this is not part of the normal name mangling in C++
+ if symbol:
+ # the anchor will be our parent
+ return symbol.parent.declaration.get_id(version, prefixed=None)
+ else:
+ res = '_'
+ if self.parameterPack:
+ res += 'Dp'
+ return res + self.param.get_id(version)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = transform(self.param)
+ if self.parameterPack:
+ res += '...'
+ return res
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ self.param.describe_signature(signode, mode, env, symbol)
+ if self.parameterPack:
+ signode += addnodes.desc_sig_punctuation('...', '...')
-class ASTTemplateParams(ASTBase):
- def __init__(self, params: list[ASTTemplateParam], requiresClause: (
- ASTRequiresClause | None)) ->None:
+class ASTTemplateParams(ASTBase):
+ def __init__(self, params: list[ASTTemplateParam],
+ requiresClause: ASTRequiresClause | None) -> None:
assert params is not None
self.params = params
self.requiresClause = requiresClause
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateParams):
return NotImplemented
- return (self.params == other.params and self.requiresClause ==
- other.requiresClause)
+ return self.params == other.params and self.requiresClause == other.requiresClause
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.params, self.requiresClause))
+ def get_id(self, version: int, excludeRequires: bool = False) -> str:
+ assert version >= 2
+ res = []
+ res.append("I")
+ res.extend(param.get_id(version) for param in self.params)
+ res.append("E")
+ if not excludeRequires and self.requiresClause:
+ res.extend(['IQ', self.requiresClause.expr.get_id(version), 'E'])
+ return ''.join(res)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append("template<")
+ res.append(", ".join(transform(a) for a in self.params))
+ res.append("> ")
+ if self.requiresClause is not None:
+ res.append(transform(self.requiresClause))
+ res.append(" ")
+ return ''.join(res)
+
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('template', 'template')
+ signode += addnodes.desc_sig_punctuation('<', '<')
+ first = True
+ for param in self.params:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ first = False
+ param.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation('>', '>')
+ if self.requiresClause is not None:
+ signode += addnodes.desc_sig_space()
+ self.requiresClause.describe_signature(signode, mode, env, symbol)
+
+ def describe_signature_as_introducer(
+ self, parentNode: desc_signature, mode: str, env: BuildEnvironment,
+ symbol: Symbol, lineSpec: bool) -> None:
+ def makeLine(parentNode: desc_signature) -> addnodes.desc_signature_line:
+ signode = addnodes.desc_signature_line()
+ parentNode += signode
+ signode.sphinx_line_type = 'templateParams'
+ return signode
+ lineNode = makeLine(parentNode)
+ lineNode += addnodes.desc_sig_keyword('template', 'template')
+ lineNode += addnodes.desc_sig_punctuation('<', '<')
+ first = True
+ for param in self.params:
+ if not first:
+ lineNode += addnodes.desc_sig_punctuation(',', ',')
+ lineNode += addnodes.desc_sig_space()
+ first = False
+ if lineSpec:
+ lineNode = makeLine(parentNode)
+ param.describe_signature(lineNode, mode, env, symbol)
+ if lineSpec and not first:
+ lineNode = makeLine(parentNode)
+ lineNode += addnodes.desc_sig_punctuation('>', '>')
+ if self.requiresClause:
+ reqNode = addnodes.desc_signature_line()
+ reqNode.sphinx_line_type = 'requiresClause'
+ parentNode += reqNode
+ self.requiresClause.describe_signature(reqNode, 'markType', env, symbol)
+
+
+# Template introducers
+################################################################################
class ASTTemplateIntroductionParameter(ASTBase):
-
- def __init__(self, identifier: ASTIdentifier, parameterPack: bool) ->None:
+ def __init__(self, identifier: ASTIdentifier, parameterPack: bool) -> None:
self.identifier = identifier
self.parameterPack = parameterPack
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateIntroductionParameter):
return NotImplemented
- return (self.identifier == other.identifier and self.parameterPack ==
- other.parameterPack)
+ return (
+ self.identifier == other.identifier
+ and self.parameterPack == other.parameterPack
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.identifier, self.parameterPack))
+ @property
+ def name(self) -> ASTNestedName:
+ id = self.get_identifier()
+ return ASTNestedName([ASTNestedNameElement(id, None)], [False], rooted=False)
+
+ @property
+ def isPack(self) -> bool:
+ return self.parameterPack
+
+ def get_identifier(self) -> ASTIdentifier:
+ return self.identifier
+
+ def get_id(
+ self, version: int, objectType: str | None = None, symbol: Symbol | None = None,
+ ) -> str:
+ assert version >= 2
+ # this is not part of the normal name mangling in C++
+ if symbol:
+ # the anchor will be our parent
+ return symbol.parent.declaration.get_id(version, prefixed=None)
+ else:
+ if self.parameterPack:
+ return 'Dp'
+ else:
+ return '0' # we need to put something
+
+ def get_id_as_arg(self, version: int) -> str:
+ assert version >= 2
+ # used for the implicit requires clause
+ res = self.identifier.get_id(version)
+ if self.parameterPack:
+ return 'sp' + res
+ else:
+ return res
-class ASTTemplateIntroduction(ASTBase):
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.parameterPack:
+ res.append('...')
+ res.append(transform(self.identifier))
+ return ''.join(res)
- def __init__(self, concept: ASTNestedName, params: list[
- ASTTemplateIntroductionParameter]) ->None:
+ def describe_signature(self, signode: TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ if self.parameterPack:
+ signode += addnodes.desc_sig_punctuation('...', '...')
+ self.identifier.describe_signature(signode, mode, env, '', '', symbol)
+
+
+class ASTTemplateIntroduction(ASTBase):
+ def __init__(self, concept: ASTNestedName,
+ params: list[ASTTemplateIntroductionParameter]) -> None:
assert len(params) > 0
self.concept = concept
self.params = params
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateIntroduction):
return NotImplemented
return self.concept == other.concept and self.params == other.params
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.concept, self.params))
+ def get_id(self, version: int) -> str:
+ assert version >= 2
+ return ''.join([
+ # first do the same as a normal template parameter list
+ "I",
+ *(param.get_id(version) for param in self.params),
+ "E",
+ # let's use X expr E, which is otherwise for constant template args
+ "X",
+ self.concept.get_id(version),
+ "I",
+ *(param.get_id_as_arg(version) for param in self.params),
+ "E",
+ "E",
+ ])
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ res.append(transform(self.concept))
+ res.append('{')
+ res.append(', '.join(transform(param) for param in self.params))
+ res.append('} ')
+ return ''.join(res)
+
+ def describe_signature_as_introducer(
+ self, parentNode: desc_signature, mode: str,
+ env: BuildEnvironment, symbol: Symbol, lineSpec: bool) -> None:
+ # Note: 'lineSpec' has no effect on template introductions.
+ signode = addnodes.desc_signature_line()
+ parentNode += signode
+ signode.sphinx_line_type = 'templateIntroduction'
+ self.concept.describe_signature(signode, 'markType', env, symbol)
+ signode += addnodes.desc_sig_punctuation('{', '{')
+ first = True
+ for param in self.params:
+ if not first:
+ signode += addnodes.desc_sig_punctuation(',', ',')
+ signode += addnodes.desc_sig_space()
+ first = False
+ param.describe_signature(signode, mode, env, symbol)
+ signode += addnodes.desc_sig_punctuation('}', '}')
+
+
+################################################################################
class ASTTemplateDeclarationPrefix(ASTBase):
-
- def __init__(self, templates: (list[ASTTemplateParams |
- ASTTemplateIntroduction] | None)) ->None:
+ def __init__(self,
+ templates: list[ASTTemplateParams | ASTTemplateIntroduction] | None) -> None:
+ # templates is None means it's an explicit instantiation of a variable
self.templates = templates
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTTemplateDeclarationPrefix):
return NotImplemented
return self.templates == other.templates
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.templates)
+ def get_requires_clause_in_last(self) -> ASTRequiresClause | None:
+ if self.templates is None:
+ return None
+ lastList = self.templates[-1]
+ if not isinstance(lastList, ASTTemplateParams):
+ return None
+ return lastList.requiresClause # which may be None
+
+ def get_id_except_requires_clause_in_last(self, version: int) -> str:
+ assert version >= 2
+ # This is not part of the Itanium ABI mangling system.
+ res = []
+ lastIndex = len(self.templates) - 1
+ for i, t in enumerate(self.templates):
+ if isinstance(t, ASTTemplateParams):
+ res.append(t.get_id(version, excludeRequires=(i == lastIndex)))
+ else:
+ res.append(t.get_id(version))
+ return ''.join(res)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return ''.join(map(transform, self.templates))
+
+ def describe_signature(self, signode: desc_signature, mode: str,
+ env: BuildEnvironment, symbol: Symbol, lineSpec: bool) -> None:
+ verify_description_mode(mode)
+ for t in self.templates:
+ t.describe_signature_as_introducer(signode, 'lastIsName', env, symbol, lineSpec)
-class ASTRequiresClause(ASTBase):
- def __init__(self, expr: ASTExpression) ->None:
+class ASTRequiresClause(ASTBase):
+ def __init__(self, expr: ASTExpression) -> None:
self.expr = expr
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTRequiresClause):
return NotImplemented
return self.expr == other.expr
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.expr)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return 'requires ' + transform(self.expr)
-class ASTDeclaration(ASTBase):
+ def describe_signature(self, signode: nodes.TextElement, mode: str,
+ env: BuildEnvironment, symbol: Symbol) -> None:
+ signode += addnodes.desc_sig_keyword('requires', 'requires')
+ signode += addnodes.desc_sig_space()
+ self.expr.describe_signature(signode, mode, env, symbol)
- def __init__(self, objectType: str, directiveType: (str | None)=None,
- visibility: (str | None)=None, templatePrefix: (
- ASTTemplateDeclarationPrefix | None)=None, declaration: Any=None,
- trailingRequiresClause: (ASTRequiresClause | None)=None, semicolon:
- bool=False) ->None:
+
+################################################################################
+################################################################################
+
+class ASTDeclaration(ASTBase):
+ def __init__(self, objectType: str, directiveType: str | None = None,
+ visibility: str | None = None,
+ templatePrefix: ASTTemplateDeclarationPrefix | None = None,
+ declaration: Any = None,
+ trailingRequiresClause: ASTRequiresClause | None = None,
+ semicolon: bool = False) -> None:
self.objectType = objectType
self.directiveType = directiveType
self.visibility = visibility
@@ -1409,31 +4236,196 @@ class ASTDeclaration(ASTBase):
self.declaration = declaration
self.trailingRequiresClause = trailingRequiresClause
self.semicolon = semicolon
+
self.symbol: Symbol | None = None
+ # set by CPPObject._add_enumerator_to_parent
self.enumeratorScopedSymbol: Symbol | None = None
+
+ # the cache assumes that by the time get_newest_id is called, no
+ # further changes will be made to this object
self._newest_id_cache: str | None = None
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTDeclaration):
return NotImplemented
- return (self.objectType == other.objectType and self.directiveType ==
- other.directiveType and self.visibility == other.visibility and
- self.templatePrefix == other.templatePrefix and self.
- declaration == other.declaration and self.
- trailingRequiresClause == other.trailingRequiresClause and self
- .semicolon == other.semicolon and self.symbol == other.symbol and
- self.enumeratorScopedSymbol == other.enumeratorScopedSymbol)
+ return (
+ self.objectType == other.objectType
+ and self.directiveType == other.directiveType
+ and self.visibility == other.visibility
+ and self.templatePrefix == other.templatePrefix
+ and self.declaration == other.declaration
+ and self.trailingRequiresClause == other.trailingRequiresClause
+ and self.semicolon == other.semicolon
+ and self.symbol == other.symbol
+ and self.enumeratorScopedSymbol == other.enumeratorScopedSymbol
+ )
+
+ def clone(self) -> ASTDeclaration:
+ templatePrefixClone = self.templatePrefix.clone() if self.templatePrefix else None
+ trailingRequiresClasueClone = self.trailingRequiresClause.clone() \
+ if self.trailingRequiresClause else None
+ return ASTDeclaration(self.objectType, self.directiveType, self.visibility,
+ templatePrefixClone,
+ self.declaration.clone(), trailingRequiresClasueClone,
+ self.semicolon)
+
+ @property
+ def name(self) -> ASTNestedName:
+ return self.declaration.name
+
+ @property
+ def function_params(self) -> list[ASTFunctionParameter]:
+ if self.objectType != 'function':
+ return None
+ return self.declaration.function_params
+
+ def get_id(self, version: int, prefixed: bool = True) -> str:
+ if version == 1:
+ if self.templatePrefix or self.trailingRequiresClause:
+ raise NoOldIdError
+ if self.objectType == 'enumerator' and self.enumeratorScopedSymbol:
+ return self.enumeratorScopedSymbol.declaration.get_id(version)
+ return self.declaration.get_id(version, self.objectType, self.symbol)
+ # version >= 2
+ if self.objectType == 'enumerator' and self.enumeratorScopedSymbol:
+ return self.enumeratorScopedSymbol.declaration.get_id(version, prefixed)
+ if prefixed:
+ res = [_id_prefix[version]]
+ else:
+ res = []
+ # (See also https://github.com/sphinx-doc/sphinx/pull/10286#issuecomment-1168102147)
+ # The first implementation of requires clauses only supported a single clause after the
+ # template prefix, and no trailing clause. It put the ID after the template parameter
+ # list, i.e.,
+ # "I" + template_parameter_list_id + "E" + "IQ" + requires_clause_id + "E"
+ # but the second implementation associates the requires clause with each list, i.e.,
+ # "I" + template_parameter_list_id + "IQ" + requires_clause_id + "E" + "E"
+ # To avoid making a new ID version, we make an exception for the last requires clause
+ # in the template prefix, and still put it in the end.
+ # As we now support trailing requires clauses we add that as if it was a conjunction.
+ if self.templatePrefix is not None:
+ res.append(self.templatePrefix.get_id_except_requires_clause_in_last(version))
+ requiresClauseInLast = self.templatePrefix.get_requires_clause_in_last()
+ else:
+ requiresClauseInLast = None
+
+ if requiresClauseInLast or self.trailingRequiresClause:
+ if version < 4:
+ raise NoOldIdError
+ res.append('IQ')
+ if requiresClauseInLast and self.trailingRequiresClause:
+ # make a conjunction of them
+ res.append('aa')
+ if requiresClauseInLast:
+ res.append(requiresClauseInLast.expr.get_id(version))
+ if self.trailingRequiresClause:
+ res.append(self.trailingRequiresClause.expr.get_id(version))
+ res.append('E')
+ res.append(self.declaration.get_id(version, self.objectType, self.symbol))
+ return ''.join(res)
+
+ def get_newest_id(self) -> str:
+ if self._newest_id_cache is None:
+ self._newest_id_cache = self.get_id(_max_id, True)
+ return self._newest_id_cache
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.visibility and self.visibility != "public":
+ res.append(self.visibility)
+ res.append(' ')
+ if self.templatePrefix:
+ res.append(transform(self.templatePrefix))
+ res.append(transform(self.declaration))
+ if self.trailingRequiresClause:
+ res.append(' ')
+ res.append(transform(self.trailingRequiresClause))
+ if self.semicolon:
+ res.append(';')
+ return ''.join(res)
+
+ def describe_signature(self, signode: desc_signature, mode: str,
+ env: BuildEnvironment, options: dict[str, bool]) -> None:
+ verify_description_mode(mode)
+ assert self.symbol
+ # The caller of the domain added a desc_signature node.
+ # Always enable multiline:
+ signode['is_multiline'] = True
+ # Put each line in a desc_signature_line node.
+ mainDeclNode = addnodes.desc_signature_line()
+ mainDeclNode.sphinx_line_type = 'declarator'
+ mainDeclNode['add_permalink'] = not self.symbol.isRedeclaration
+
+ if self.templatePrefix:
+ self.templatePrefix.describe_signature(signode, mode, env,
+ symbol=self.symbol,
+ lineSpec=options.get('tparam-line-spec'))
+ signode += mainDeclNode
+ if self.visibility and self.visibility != "public":
+ mainDeclNode += addnodes.desc_sig_keyword(self.visibility, self.visibility)
+ mainDeclNode += addnodes.desc_sig_space()
+ if self.objectType == 'type':
+ prefix = self.declaration.get_type_declaration_prefix()
+ mainDeclNode += addnodes.desc_sig_keyword(prefix, prefix)
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.objectType == 'concept':
+ mainDeclNode += addnodes.desc_sig_keyword('concept', 'concept')
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.objectType in {'member', 'function'}:
+ pass
+ elif self.objectType == 'class':
+ assert self.directiveType in ('class', 'struct')
+ mainDeclNode += addnodes.desc_sig_keyword(self.directiveType, self.directiveType)
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.objectType == 'union':
+ mainDeclNode += addnodes.desc_sig_keyword('union', 'union')
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.objectType == 'enum':
+ mainDeclNode += addnodes.desc_sig_keyword('enum', 'enum')
+ mainDeclNode += addnodes.desc_sig_space()
+ if self.directiveType == 'enum-class':
+ mainDeclNode += addnodes.desc_sig_keyword('class', 'class')
+ mainDeclNode += addnodes.desc_sig_space()
+ elif self.directiveType == 'enum-struct':
+ mainDeclNode += addnodes.desc_sig_keyword('struct', 'struct')
+ mainDeclNode += addnodes.desc_sig_space()
+ else:
+ assert self.directiveType == 'enum', self.directiveType
+ elif self.objectType == 'enumerator':
+ mainDeclNode += addnodes.desc_sig_keyword('enumerator', 'enumerator')
+ mainDeclNode += addnodes.desc_sig_space()
+ else:
+ raise AssertionError(self.objectType)
+ self.declaration.describe_signature(mainDeclNode, mode, env, self.symbol)
+ lastDeclNode = mainDeclNode
+ if self.trailingRequiresClause:
+ trailingReqNode = addnodes.desc_signature_line()
+ trailingReqNode.sphinx_line_type = 'trailingRequiresClause'
+ signode.append(trailingReqNode)
+ lastDeclNode = trailingReqNode
+ self.trailingRequiresClause.describe_signature(
+ trailingReqNode, 'markType', env, self.symbol)
+ if self.semicolon:
+ lastDeclNode += addnodes.desc_sig_punctuation(';', ';')
class ASTNamespace(ASTBase):
-
- def __init__(self, nestedName: ASTNestedName, templatePrefix:
- ASTTemplateDeclarationPrefix) ->None:
+ def __init__(self, nestedName: ASTNestedName,
+ templatePrefix: ASTTemplateDeclarationPrefix) -> None:
self.nestedName = nestedName
self.templatePrefix = templatePrefix
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTNamespace):
return NotImplemented
- return (self.nestedName == other.nestedName and self.templatePrefix ==
- other.templatePrefix)
+ return (
+ self.nestedName == other.nestedName
+ and self.templatePrefix == other.templatePrefix
+ )
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ res = []
+ if self.templatePrefix:
+ res.append(transform(self.templatePrefix))
+ res.append(transform(self.nestedName))
+ return ''.join(res)
diff --git a/sphinx/domains/cpp/_ids.py b/sphinx/domains/cpp/_ids.py
index aceb10c93..ee8eb4920 100644
--- a/sphinx/domains/cpp/_ids.py
+++ b/sphinx/domains/cpp/_ids.py
@@ -249,54 +249,54 @@ namespace_object:
grammar:
nested-name
"""
+
from __future__ import annotations
+
import re
-udl_identifier_re = re.compile(
- """
- [a-zA-Z_][a-zA-Z0-9_]*\\b # note, no word boundary in the beginning
-"""
- , re.VERBOSE)
-_string_re = re.compile(
- '[LuU8]?(\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'|"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)")'
- , re.DOTALL)
-_visibility_re = re.compile('\\b(public|private|protected)\\b')
-_operator_re = re.compile(
- """
- \\[\\s*\\]
- | \\(\\s*\\)
- | \\+\\+ | --
- | ->\\*? | \\,
- | (<<|>>)=? | && | \\|\\|
+
+udl_identifier_re = re.compile(r'''
+ [a-zA-Z_][a-zA-Z0-9_]*\b # note, no word boundary in the beginning
+''', re.VERBOSE)
+_string_re = re.compile(r"[LuU8]?('([^'\\]*(?:\\.[^'\\]*)*)'"
+ r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.DOTALL)
+_visibility_re = re.compile(r'\b(public|private|protected)\b')
+_operator_re = re.compile(r'''
+ \[\s*\]
+ | \(\s*\)
+ | \+\+ | --
+ | ->\*? | \,
+ | (<<|>>)=? | && | \|\|
| <=>
| [!<>=/*%+|&^~-]=?
- | (\\b(and|and_eq|bitand|bitor|compl|not|not_eq|or|or_eq|xor|xor_eq)\\b)
-"""
- , re.VERBOSE)
-_fold_operator_re = re.compile(
- """
- ->\\* | \\.\\* | \\,
- | (<<|>>)=? | && | \\|\\|
+ | (\b(and|and_eq|bitand|bitor|compl|not|not_eq|or|or_eq|xor|xor_eq)\b)
+''', re.VERBOSE)
+_fold_operator_re = re.compile(r'''
+ ->\* | \.\* | \,
+ | (<<|>>)=? | && | \|\|
| !=
| [<>=/*%+|&^~-]=?
-"""
- , re.VERBOSE)
-_keywords = ['alignas', 'alignof', 'and', 'and_eq', 'asm', 'auto', 'bitand',
- 'bitor', 'bool', 'break', 'case', 'catch', 'char', 'char8_t',
- 'char16_t', 'char32_t', 'class', 'compl', 'concept', 'const',
- 'consteval', 'constexpr', 'constinit', 'const_cast', 'continue',
+''', re.VERBOSE)
+# see https://en.cppreference.com/w/cpp/keyword
+_keywords = [
+ 'alignas', 'alignof', 'and', 'and_eq', 'asm', 'auto', 'bitand', 'bitor',
+ 'bool', 'break', 'case', 'catch', 'char', 'char8_t', 'char16_t', 'char32_t',
+ 'class', 'compl', 'concept', 'const', 'consteval', 'constexpr', 'constinit',
+ 'const_cast', 'continue',
'decltype', 'default', 'delete', 'do', 'double', 'dynamic_cast', 'else',
- 'enum', 'explicit', 'export', 'extern', 'false', 'float', 'for',
- 'friend', 'goto', 'if', 'inline', 'int', 'long', 'mutable', 'namespace',
- 'new', 'noexcept', 'not', 'not_eq', 'nullptr', 'operator', 'or',
- 'or_eq', 'private', 'protected', 'public', 'register',
- 'reinterpret_cast', 'requires', 'return', 'short', 'signed', 'sizeof',
- 'static', 'static_assert', 'static_cast', 'struct', 'switch',
- 'template', 'this', 'thread_local', 'throw', 'true', 'try', 'typedef',
- 'typeid', 'typename', 'union', 'unsigned', 'using', 'virtual', 'void',
- 'volatile', 'wchar_t', 'while', 'xor', 'xor_eq']
-_simple_type_specifiers_re = re.compile(
- """
- \\b(
+ 'enum', 'explicit', 'export', 'extern', 'false', 'float', 'for', 'friend',
+ 'goto', 'if', 'inline', 'int', 'long', 'mutable', 'namespace', 'new',
+ 'noexcept', 'not', 'not_eq', 'nullptr', 'operator', 'or', 'or_eq',
+ 'private', 'protected', 'public', 'register', 'reinterpret_cast',
+ 'requires', 'return', 'short', 'signed', 'sizeof', 'static',
+ 'static_assert', 'static_cast', 'struct', 'switch', 'template', 'this',
+ 'thread_local', 'throw', 'true', 'try', 'typedef', 'typeid', 'typename',
+ 'union', 'unsigned', 'using', 'virtual', 'void', 'volatile', 'wchar_t',
+ 'while', 'xor', 'xor_eq',
+]
+
+
+_simple_type_specifiers_re = re.compile(r"""
+ \b(
auto|void|bool
|signed|unsigned
|short|long
@@ -306,72 +306,232 @@ _simple_type_specifiers_re = re.compile(
|float|double
|__float80|_Float64x|__float128|_Float128 # extension
|_Complex|_Imaginary # extension
- )\\b
-"""
- , re.VERBOSE)
+ )\b
+""", re.VERBOSE)
+
_max_id = 4
_id_prefix = [None, '', '_CPPv2', '_CPPv3', '_CPPv4']
-_id_fundamental_v1 = {'char': 'c', 'signed char': 'c', 'unsigned char': 'C',
- 'int': 'i', 'signed int': 'i', 'unsigned int': 'U', 'long': 'l',
- 'signed long': 'l', 'unsigned long': 'L', 'bool': 'b'}
-_id_shorthands_v1 = {'std::string': 'ss', 'std::ostream': 'os',
- 'std::istream': 'is', 'std::iostream': 'ios', 'std::vector': 'v',
- 'std::map': 'm'}
-_id_operator_v1 = {'new': 'new-operator', 'new[]': 'new-array-operator',
- 'delete': 'delete-operator', 'delete[]': 'delete-array-operator', '~':
- 'inv-operator', '+': 'add-operator', '-': 'sub-operator', '*':
- 'mul-operator', '/': 'div-operator', '%': 'mod-operator', '&':
- 'and-operator', '|': 'or-operator', '^': 'xor-operator', '=':
- 'assign-operator', '+=': 'add-assign-operator', '-=':
- 'sub-assign-operator', '*=': 'mul-assign-operator', '/=':
- 'div-assign-operator', '%=': 'mod-assign-operator', '&=':
- 'and-assign-operator', '|=': 'or-assign-operator', '^=':
- 'xor-assign-operator', '<<': 'lshift-operator', '>>': 'rshift-operator',
- '<<=': 'lshift-assign-operator', '>>=': 'rshift-assign-operator', '==':
- 'eq-operator', '!=': 'neq-operator', '<': 'lt-operator', '>':
- 'gt-operator', '<=': 'lte-operator', '>=': 'gte-operator', '!':
- 'not-operator', '&&': 'sand-operator', '||': 'sor-operator', '++':
- 'inc-operator', '--': 'dec-operator', ',': 'comma-operator', '->*':
- 'pointer-by-pointer-operator', '->': 'pointer-operator', '()':
- 'call-operator', '[]': 'subscript-operator'}
-_id_fundamental_v2 = {'void': 'v', 'bool': 'b', 'char': 'c', 'signed char':
- 'a', 'unsigned char': 'h', 'wchar_t': 'w', 'char32_t': 'Di', 'char16_t':
- 'Ds', 'char8_t': 'Du', 'short': 's', 'short int': 's', 'signed short':
- 's', 'signed short int': 's', 'unsigned short': 't',
- 'unsigned short int': 't', 'int': 'i', 'signed': 'i', 'signed int': 'i',
- 'unsigned': 'j', 'unsigned int': 'j', 'long': 'l', 'long int': 'l',
- 'signed long': 'l', 'signed long int': 'l', 'unsigned long': 'm',
- 'unsigned long int': 'm', 'long long': 'x', 'long long int': 'x',
- 'signed long long': 'x', 'signed long long int': 'x', '__int64': 'x',
- 'unsigned long long': 'y', 'unsigned long long int': 'y', '__int128':
- 'n', 'signed __int128': 'n', 'unsigned __int128': 'o', 'float': 'f',
- 'double': 'd', 'long double': 'e', '__float80': 'e', '_Float64x': 'e',
- '__float128': 'g', '_Float128': 'g', '_Complex float': 'Cf',
- '_Complex double': 'Cd', '_Complex long double': 'Ce',
- '_Imaginary float': 'f', '_Imaginary double': 'd',
- '_Imaginary long double': 'e', 'auto': 'Da', 'decltype(auto)': 'Dc',
- 'std::nullptr_t': 'Dn'}
-_id_operator_v2 = {'new': 'nw', 'new[]': 'na', 'delete': 'dl', 'delete[]':
- 'da', '~': 'co', 'compl': 'co', '+': 'pl', '-': 'mi', '*': 'ml', '/':
- 'dv', '%': 'rm', '&': 'an', 'bitand': 'an', '|': 'or', 'bitor': 'or',
- '^': 'eo', 'xor': 'eo', '=': 'aS', '+=': 'pL', '-=': 'mI', '*=': 'mL',
- '/=': 'dV', '%=': 'rM', '&=': 'aN', 'and_eq': 'aN', '|=': 'oR', 'or_eq':
- 'oR', '^=': 'eO', 'xor_eq': 'eO', '<<': 'ls', '>>': 'rs', '<<=': 'lS',
- '>>=': 'rS', '==': 'eq', '!=': 'ne', 'not_eq': 'ne', '<': 'lt', '>':
- 'gt', '<=': 'le', '>=': 'ge', '<=>': 'ss', '!': 'nt', 'not': 'nt', '&&':
- 'aa', 'and': 'aa', '||': 'oo', 'or': 'oo', '++': 'pp', '--': 'mm', ',':
- 'cm', '->*': 'pm', '->': 'pt', '()': 'cl', '[]': 'ix', '.*': 'ds', '?':
- 'qu'}
-_id_operator_unary_v2 = {'++': 'pp_', '--': 'mm_', '*': 'de', '&': 'ad',
- '+': 'ps', '-': 'ng', '!': 'nt', 'not': 'nt', '~': 'co', 'compl': 'co'}
-_id_char_from_prefix: dict[str | None, str] = {None: 'c', 'u8': 'c', 'u':
- 'Ds', 'U': 'Di', 'L': 'w'}
-_expression_bin_ops = [['||', 'or'], ['&&', 'and'], ['|', 'bitor'], ['^',
- 'xor'], ['&', 'bitand'], ['==', '!=', 'not_eq'], ['<=>', '<=', '>=',
- '<', '>'], ['<<', '>>'], ['+', '-'], ['*', '/', '%'], ['.*', '->*']]
-_expression_unary_ops = ['++', '--', '*', '&', '+', '-', '!', 'not', '~',
- 'compl']
-_expression_assignment_ops = ['=', '*=', '/=', '%=', '+=', '-=', '>>=',
- '<<=', '&=', 'and_eq', '^=', '|=', 'xor_eq', 'or_eq']
-_id_explicit_cast = {'dynamic_cast': 'dc', 'static_cast': 'sc',
- 'const_cast': 'cc', 'reinterpret_cast': 'rc'}
+# Ids are used in lookup keys which are used across pickled files,
+# so when _max_id changes, make sure to update the ENV_VERSION.
+
+# ------------------------------------------------------------------------------
+# Id v1 constants
+# ------------------------------------------------------------------------------
+
+_id_fundamental_v1 = {
+ 'char': 'c',
+ 'signed char': 'c',
+ 'unsigned char': 'C',
+ 'int': 'i',
+ 'signed int': 'i',
+ 'unsigned int': 'U',
+ 'long': 'l',
+ 'signed long': 'l',
+ 'unsigned long': 'L',
+ 'bool': 'b',
+}
+_id_shorthands_v1 = {
+ 'std::string': 'ss',
+ 'std::ostream': 'os',
+ 'std::istream': 'is',
+ 'std::iostream': 'ios',
+ 'std::vector': 'v',
+ 'std::map': 'm',
+}
+_id_operator_v1 = {
+ 'new': 'new-operator',
+ 'new[]': 'new-array-operator',
+ 'delete': 'delete-operator',
+ 'delete[]': 'delete-array-operator',
+ # the arguments will make the difference between unary and binary
+ # '+(unary)' : 'ps',
+ # '-(unary)' : 'ng',
+ # '&(unary)' : 'ad',
+ # '*(unary)' : 'de',
+ '~': 'inv-operator',
+ '+': 'add-operator',
+ '-': 'sub-operator',
+ '*': 'mul-operator',
+ '/': 'div-operator',
+ '%': 'mod-operator',
+ '&': 'and-operator',
+ '|': 'or-operator',
+ '^': 'xor-operator',
+ '=': 'assign-operator',
+ '+=': 'add-assign-operator',
+ '-=': 'sub-assign-operator',
+ '*=': 'mul-assign-operator',
+ '/=': 'div-assign-operator',
+ '%=': 'mod-assign-operator',
+ '&=': 'and-assign-operator',
+ '|=': 'or-assign-operator',
+ '^=': 'xor-assign-operator',
+ '<<': 'lshift-operator',
+ '>>': 'rshift-operator',
+ '<<=': 'lshift-assign-operator',
+ '>>=': 'rshift-assign-operator',
+ '==': 'eq-operator',
+ '!=': 'neq-operator',
+ '<': 'lt-operator',
+ '>': 'gt-operator',
+ '<=': 'lte-operator',
+ '>=': 'gte-operator',
+ '!': 'not-operator',
+ '&&': 'sand-operator',
+ '||': 'sor-operator',
+ '++': 'inc-operator',
+ '--': 'dec-operator',
+ ',': 'comma-operator',
+ '->*': 'pointer-by-pointer-operator',
+ '->': 'pointer-operator',
+ '()': 'call-operator',
+ '[]': 'subscript-operator',
+}
+
+# ------------------------------------------------------------------------------
+# Id v > 1 constants
+# ------------------------------------------------------------------------------
+
+_id_fundamental_v2 = {
+ # not all of these are actually parsed as fundamental types, TODO: do that
+ 'void': 'v',
+ 'bool': 'b',
+ 'char': 'c',
+ 'signed char': 'a',
+ 'unsigned char': 'h',
+ 'wchar_t': 'w',
+ 'char32_t': 'Di',
+ 'char16_t': 'Ds',
+ 'char8_t': 'Du',
+ 'short': 's',
+ 'short int': 's',
+ 'signed short': 's',
+ 'signed short int': 's',
+ 'unsigned short': 't',
+ 'unsigned short int': 't',
+ 'int': 'i',
+ 'signed': 'i',
+ 'signed int': 'i',
+ 'unsigned': 'j',
+ 'unsigned int': 'j',
+ 'long': 'l',
+ 'long int': 'l',
+ 'signed long': 'l',
+ 'signed long int': 'l',
+ 'unsigned long': 'm',
+ 'unsigned long int': 'm',
+ 'long long': 'x',
+ 'long long int': 'x',
+ 'signed long long': 'x',
+ 'signed long long int': 'x',
+ '__int64': 'x',
+ 'unsigned long long': 'y',
+ 'unsigned long long int': 'y',
+ '__int128': 'n',
+ 'signed __int128': 'n',
+ 'unsigned __int128': 'o',
+ 'float': 'f',
+ 'double': 'd',
+ 'long double': 'e',
+ '__float80': 'e', '_Float64x': 'e',
+ '__float128': 'g', '_Float128': 'g',
+ '_Complex float': 'Cf',
+ '_Complex double': 'Cd',
+ '_Complex long double': 'Ce',
+ '_Imaginary float': 'f',
+ '_Imaginary double': 'd',
+ '_Imaginary long double': 'e',
+ 'auto': 'Da',
+ 'decltype(auto)': 'Dc',
+ 'std::nullptr_t': 'Dn',
+}
+_id_operator_v2 = {
+ 'new': 'nw',
+ 'new[]': 'na',
+ 'delete': 'dl',
+ 'delete[]': 'da',
+ # the arguments will make the difference between unary and binary
+ # in operator definitions
+ # '+(unary)' : 'ps',
+ # '-(unary)' : 'ng',
+ # '&(unary)' : 'ad',
+ # '*(unary)' : 'de',
+ '~': 'co', 'compl': 'co',
+ '+': 'pl',
+ '-': 'mi',
+ '*': 'ml',
+ '/': 'dv',
+ '%': 'rm',
+ '&': 'an', 'bitand': 'an',
+ '|': 'or', 'bitor': 'or',
+ '^': 'eo', 'xor': 'eo',
+ '=': 'aS',
+ '+=': 'pL',
+ '-=': 'mI',
+ '*=': 'mL',
+ '/=': 'dV',
+ '%=': 'rM',
+ '&=': 'aN', 'and_eq': 'aN',
+ '|=': 'oR', 'or_eq': 'oR',
+ '^=': 'eO', 'xor_eq': 'eO',
+ '<<': 'ls',
+ '>>': 'rs',
+ '<<=': 'lS',
+ '>>=': 'rS',
+ '==': 'eq',
+ '!=': 'ne', 'not_eq': 'ne',
+ '<': 'lt',
+ '>': 'gt',
+ '<=': 'le',
+ '>=': 'ge',
+ '<=>': 'ss',
+ '!': 'nt', 'not': 'nt',
+ '&&': 'aa', 'and': 'aa',
+ '||': 'oo', 'or': 'oo',
+ '++': 'pp',
+ '--': 'mm',
+ ',': 'cm',
+ '->*': 'pm',
+ '->': 'pt',
+ '()': 'cl',
+ '[]': 'ix',
+ '.*': 'ds', # this one is not overloadable, but we need it for expressions
+ '?': 'qu',
+}
+_id_operator_unary_v2 = {
+ '++': 'pp_',
+ '--': 'mm_',
+ '*': 'de',
+ '&': 'ad',
+ '+': 'ps',
+ '-': 'ng',
+ '!': 'nt', 'not': 'nt',
+ '~': 'co', 'compl': 'co',
+}
+_id_char_from_prefix: dict[str | None, str] = {
+ None: 'c', 'u8': 'c',
+ 'u': 'Ds', 'U': 'Di', 'L': 'w',
+}
+# these are ordered by preceedence
+_expression_bin_ops = [
+ ['||', 'or'],
+ ['&&', 'and'],
+ ['|', 'bitor'],
+ ['^', 'xor'],
+ ['&', 'bitand'],
+ ['==', '!=', 'not_eq'],
+ ['<=>', '<=', '>=', '<', '>'],
+ ['<<', '>>'],
+ ['+', '-'],
+ ['*', '/', '%'],
+ ['.*', '->*'],
+]
+_expression_unary_ops = ["++", "--", "*", "&", "+", "-", "!", "not", "~", "compl"]
+_expression_assignment_ops = ["=", "*=", "/=", "%=", "+=", "-=",
+ ">>=", "<<=", "&=", "and_eq", "^=", "|=", "xor_eq", "or_eq"]
+_id_explicit_cast = {
+ 'dynamic_cast': 'dc',
+ 'static_cast': 'sc',
+ 'const_cast': 'cc',
+ 'reinterpret_cast': 'rc',
+}
diff --git a/sphinx/domains/cpp/_parser.py b/sphinx/domains/cpp/_parser.py
index f09b4224d..d0d70221a 100644
--- a/sphinx/domains/cpp/_parser.py
+++ b/sphinx/domains/cpp/_parser.py
@@ -1,28 +1,2118 @@
from __future__ import annotations
+
import re
from typing import TYPE_CHECKING, Any
-from sphinx.domains.cpp._ast import ASTAlignofExpr, ASTArray, ASTAssignmentExpr, ASTBaseClass, ASTBinOpExpr, ASTBooleanLiteral, ASTBracedInitList, ASTCastExpr, ASTCharLiteral, ASTClass, ASTCommaExpr, ASTConcept, ASTConditionalExpr, ASTDeclaration, ASTDeclarator, ASTDeclaratorMemPtr, ASTDeclaratorNameBitField, ASTDeclaratorNameParamQual, ASTDeclaratorParamPack, ASTDeclaratorParen, ASTDeclaratorPtr, ASTDeclaratorRef, ASTDeclSpecs, ASTDeclSpecsSimple, ASTDeleteExpr, ASTEnum, ASTEnumerator, ASTExplicitCast, ASTExplicitSpec, ASTExpression, ASTFallbackExpr, ASTFoldExpr, ASTFunctionParameter, ASTIdentifier, ASTIdExpression, ASTInitializer, ASTLiteral, ASTNamespace, ASTNestedName, ASTNestedNameElement, ASTNewExpr, ASTNoexceptExpr, ASTNoexceptSpec, ASTNumberLiteral, ASTOperator, ASTOperatorBuildIn, ASTOperatorLiteral, ASTOperatorType, ASTPackExpansionExpr, ASTParametersQualifiers, ASTParenExpr, ASTParenExprList, ASTPointerLiteral, ASTPostfixArray, ASTPostfixCallExpr, ASTPostfixDec, ASTPostfixExpr, ASTPostfixInc, ASTPostfixMember, ASTPostfixMemberOfPointer, ASTPostfixOp, ASTRequiresClause, ASTSizeofExpr, ASTSizeofParamPack, ASTSizeofType, ASTStringLiteral, ASTTemplateArgConstant, ASTTemplateArgs, ASTTemplateDeclarationPrefix, ASTTemplateIntroduction, ASTTemplateIntroductionParameter, ASTTemplateKeyParamPackIdDefault, ASTTemplateParam, ASTTemplateParamConstrainedTypeWithInit, ASTTemplateParamNonType, ASTTemplateParams, ASTTemplateParamTemplateType, ASTTemplateParamType, ASTThisLiteral, ASTTrailingTypeSpec, ASTTrailingTypeSpecDecltype, ASTTrailingTypeSpecDecltypeAuto, ASTTrailingTypeSpecFundamental, ASTTrailingTypeSpecName, ASTType, ASTTypeId, ASTTypeUsing, ASTTypeWithInit, ASTUnaryOpExpr, ASTUnion, ASTUserDefinedLiteral
-from sphinx.domains.cpp._ids import _expression_assignment_ops, _expression_bin_ops, _expression_unary_ops, _fold_operator_re, _id_explicit_cast, _keywords, _operator_re, _simple_type_specifiers_re, _string_re, _visibility_re, udl_identifier_re
+
+from sphinx.domains.cpp._ast import (
+ ASTAlignofExpr,
+ ASTArray,
+ ASTAssignmentExpr,
+ ASTBaseClass,
+ ASTBinOpExpr,
+ ASTBooleanLiteral,
+ ASTBracedInitList,
+ ASTCastExpr,
+ ASTCharLiteral,
+ ASTClass,
+ ASTCommaExpr,
+ ASTConcept,
+ ASTConditionalExpr,
+ ASTDeclaration,
+ ASTDeclarator,
+ ASTDeclaratorMemPtr,
+ ASTDeclaratorNameBitField,
+ ASTDeclaratorNameParamQual,
+ ASTDeclaratorParamPack,
+ ASTDeclaratorParen,
+ ASTDeclaratorPtr,
+ ASTDeclaratorRef,
+ ASTDeclSpecs,
+ ASTDeclSpecsSimple,
+ ASTDeleteExpr,
+ ASTEnum,
+ ASTEnumerator,
+ ASTExplicitCast,
+ ASTExplicitSpec,
+ ASTExpression,
+ ASTFallbackExpr,
+ ASTFoldExpr,
+ ASTFunctionParameter,
+ ASTIdentifier,
+ ASTIdExpression,
+ ASTInitializer,
+ ASTLiteral,
+ ASTNamespace,
+ ASTNestedName,
+ ASTNestedNameElement,
+ ASTNewExpr,
+ ASTNoexceptExpr,
+ ASTNoexceptSpec,
+ ASTNumberLiteral,
+ ASTOperator,
+ ASTOperatorBuildIn,
+ ASTOperatorLiteral,
+ ASTOperatorType,
+ ASTPackExpansionExpr,
+ ASTParametersQualifiers,
+ ASTParenExpr,
+ ASTParenExprList,
+ ASTPointerLiteral,
+ ASTPostfixArray,
+ ASTPostfixCallExpr,
+ ASTPostfixDec,
+ ASTPostfixExpr,
+ ASTPostfixInc,
+ ASTPostfixMember,
+ ASTPostfixMemberOfPointer,
+ ASTPostfixOp,
+ ASTRequiresClause,
+ ASTSizeofExpr,
+ ASTSizeofParamPack,
+ ASTSizeofType,
+ ASTStringLiteral,
+ ASTTemplateArgConstant,
+ ASTTemplateArgs,
+ ASTTemplateDeclarationPrefix,
+ ASTTemplateIntroduction,
+ ASTTemplateIntroductionParameter,
+ ASTTemplateKeyParamPackIdDefault,
+ ASTTemplateParam,
+ ASTTemplateParamConstrainedTypeWithInit,
+ ASTTemplateParamNonType,
+ ASTTemplateParams,
+ ASTTemplateParamTemplateType,
+ ASTTemplateParamType,
+ ASTThisLiteral,
+ ASTTrailingTypeSpec,
+ ASTTrailingTypeSpecDecltype,
+ ASTTrailingTypeSpecDecltypeAuto,
+ ASTTrailingTypeSpecFundamental,
+ ASTTrailingTypeSpecName,
+ ASTType,
+ ASTTypeId,
+ ASTTypeUsing,
+ ASTTypeWithInit,
+ ASTUnaryOpExpr,
+ ASTUnion,
+ ASTUserDefinedLiteral,
+)
+from sphinx.domains.cpp._ids import (
+ _expression_assignment_ops,
+ _expression_bin_ops,
+ _expression_unary_ops,
+ _fold_operator_re,
+ _id_explicit_cast,
+ _keywords,
+ _operator_re,
+ _simple_type_specifiers_re,
+ _string_re,
+ _visibility_re,
+ udl_identifier_re,
+)
from sphinx.util import logging
-from sphinx.util.cfamily import ASTAttributeList, BaseParser, DefinitionError, UnsupportedMultiCharacterCharLiteral, binary_literal_re, char_literal_re, float_literal_re, float_literal_suffix_re, hex_literal_re, identifier_re, integer_literal_re, integers_literal_suffix_re, octal_literal_re
+from sphinx.util.cfamily import (
+ ASTAttributeList,
+ BaseParser,
+ DefinitionError,
+ UnsupportedMultiCharacterCharLiteral,
+ binary_literal_re,
+ char_literal_re,
+ float_literal_re,
+ float_literal_suffix_re,
+ hex_literal_re,
+ identifier_re,
+ integer_literal_re,
+ integers_literal_suffix_re,
+ octal_literal_re,
+)
+
if TYPE_CHECKING:
from collections.abc import Callable, Sequence
+
logger = logging.getLogger(__name__)
class DefinitionParser(BaseParser):
+ @property
+ def language(self) -> str:
+ return 'C++'
+
+ @property
+ def id_attributes(self) -> Sequence[str]:
+ return self.config.cpp_id_attributes
+
+ @property
+ def paren_attributes(self) -> Sequence[str]:
+ return self.config.cpp_paren_attributes
+
+ def _parse_string(self) -> str:
+ if self.current_char != '"':
+ return None
+ startPos = self.pos
+ self.pos += 1
+ escape = False
+ while True:
+ if self.eof:
+ self.fail("Unexpected end during inside string.")
+ elif self.current_char == '"' and not escape:
+ self.pos += 1
+ break
+ elif self.current_char == '\\':
+ escape = True
+ else:
+ escape = False
+ self.pos += 1
+ return self.definition[startPos:self.pos]
+
+ def _parse_literal(self) -> ASTLiteral:
+ # -> integer-literal
+ # | character-literal
+ # | floating-literal
+ # | string-literal
+ # | boolean-literal -> "false" | "true"
+ # | pointer-literal -> "nullptr"
+ # | user-defined-literal
+
+ def _udl(literal: ASTLiteral) -> ASTLiteral:
+ if not self.match(udl_identifier_re):
+ return literal
+ # hmm, should we care if it's a keyword?
+ # it looks like GCC does not disallow keywords
+ ident = ASTIdentifier(self.matched_text)
+ return ASTUserDefinedLiteral(literal, ident)
+
+ self.skip_ws()
+ if self.skip_word('nullptr'):
+ return ASTPointerLiteral()
+ if self.skip_word('true'):
+ return ASTBooleanLiteral(True)
+ if self.skip_word('false'):
+ return ASTBooleanLiteral(False)
+ pos = self.pos
+ if self.match(float_literal_re):
+ hasSuffix = self.match(float_literal_suffix_re)
+ floatLit = ASTNumberLiteral(self.definition[pos:self.pos])
+ if hasSuffix:
+ return floatLit
+ else:
+ return _udl(floatLit)
+ for regex in (binary_literal_re, hex_literal_re,
+ integer_literal_re, octal_literal_re):
+ if self.match(regex):
+ hasSuffix = self.match(integers_literal_suffix_re)
+ intLit = ASTNumberLiteral(self.definition[pos:self.pos])
+ if hasSuffix:
+ return intLit
+ else:
+ return _udl(intLit)
+
+ string = self._parse_string()
+ if string is not None:
+ return _udl(ASTStringLiteral(string))
+
+ # character-literal
+ if self.match(char_literal_re):
+ prefix = self.last_match.group(1) # may be None when no prefix
+ data = self.last_match.group(2)
+ try:
+ charLit = ASTCharLiteral(prefix, data)
+ except UnicodeDecodeError as e:
+ self.fail("Can not handle character literal. Internal error was: %s" % e)
+ except UnsupportedMultiCharacterCharLiteral:
+ self.fail("Can not handle character literal"
+ " resulting in multiple decoded characters.")
+ return _udl(charLit)
+ return None
+
+ def _parse_fold_or_paren_expression(self) -> ASTExpression | None:
+ # "(" expression ")"
+ # fold-expression
+ # -> ( cast-expression fold-operator ... )
+ # | ( ... fold-operator cast-expression )
+ # | ( cast-expression fold-operator ... fold-operator cast-expression
+ if self.current_char != '(':
+ return None
+ self.pos += 1
+ self.skip_ws()
+ if self.skip_string_and_ws("..."):
+ # ( ... fold-operator cast-expression )
+ if not self.match(_fold_operator_re):
+ self.fail("Expected fold operator after '...' in fold expression.")
+ op = self.matched_text
+ rightExpr = self._parse_cast_expression()
+ if not self.skip_string(')'):
+ self.fail("Expected ')' in end of fold expression.")
+ return ASTFoldExpr(None, op, rightExpr)
+ # try first parsing a unary right fold, or a binary fold
+ pos = self.pos
+ try:
+ self.skip_ws()
+ leftExpr = self._parse_cast_expression()
+ self.skip_ws()
+ if not self.match(_fold_operator_re):
+ self.fail("Expected fold operator after left expression in fold expression.")
+ op = self.matched_text
+ self.skip_ws()
+ if not self.skip_string_and_ws('...'):
+ self.fail("Expected '...' after fold operator in fold expression.")
+ except DefinitionError as eFold:
+ self.pos = pos
+ # fall back to a paren expression
+ try:
+ res = self._parse_expression()
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expected ')' in end of parenthesized expression.")
+ except DefinitionError as eExpr:
+ raise self._make_multi_error([
+ (eFold, "If fold expression"),
+ (eExpr, "If parenthesized expression"),
+ ], "Error in fold expression or parenthesized expression.") from eExpr
+ return ASTParenExpr(res)
+ # now it definitely is a fold expression
+ if self.skip_string(')'):
+ return ASTFoldExpr(leftExpr, op, None)
+ if not self.match(_fold_operator_re):
+ self.fail("Expected fold operator or ')' after '...' in fold expression.")
+ if op != self.matched_text:
+ self.fail("Operators are different in binary fold: '%s' and '%s'."
+ % (op, self.matched_text))
+ rightExpr = self._parse_cast_expression()
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expected ')' to end binary fold expression.")
+ return ASTFoldExpr(leftExpr, op, rightExpr)
+
+ def _parse_primary_expression(self) -> ASTExpression:
+ # literal
+ # "this"
+ # lambda-expression
+ # "(" expression ")"
+ # fold-expression
+ # id-expression -> we parse this with _parse_nested_name
+ self.skip_ws()
+ res: ASTExpression = self._parse_literal()
+ if res is not None:
+ return res
+ self.skip_ws()
+ if self.skip_word("this"):
+ return ASTThisLiteral()
+ # TODO: try lambda expression
+ res = self._parse_fold_or_paren_expression()
+ if res is not None:
+ return res
+ nn = self._parse_nested_name()
+ if nn is not None:
+ return ASTIdExpression(nn)
+ return None
+
+ def _parse_initializer_list(self, name: str, open: str, close: str,
+ ) -> tuple[list[ASTExpression | ASTBracedInitList],
+ bool]:
+ # Parse open and close with the actual initializer-list in between
+ # -> initializer-clause '...'[opt]
+ # | initializer-list ',' initializer-clause '...'[opt]
+ self.skip_ws()
+ if not self.skip_string_and_ws(open):
+ return None, None
+ if self.skip_string(close):
+ return [], False
+
+ exprs: list[ASTExpression | ASTBracedInitList] = []
+ trailingComma = False
+ while True:
+ self.skip_ws()
+ expr = self._parse_initializer_clause()
+ self.skip_ws()
+ if self.skip_string('...'):
+ exprs.append(ASTPackExpansionExpr(expr))
+ else:
+ exprs.append(expr)
+ self.skip_ws()
+ if self.skip_string(close):
+ break
+ if not self.skip_string_and_ws(','):
+ self.fail(f"Error in {name}, expected ',' or '{close}'.")
+ if self.current_char == close == '}':
+ self.pos += 1
+ trailingComma = True
+ break
+ return exprs, trailingComma
+
+ def _parse_paren_expression_list(self) -> ASTParenExprList:
+ # -> '(' expression-list ')'
+ # though, we relax it to also allow empty parens
+ # as it's needed in some cases
+ #
+ # expression-list
+ # -> initializer-list
+ exprs, trailingComma = self._parse_initializer_list("parenthesized expression-list",
+ '(', ')')
+ if exprs is None:
+ return None
+ return ASTParenExprList(exprs)
+
+ def _parse_initializer_clause(self) -> ASTExpression | ASTBracedInitList:
+ bracedInitList = self._parse_braced_init_list()
+ if bracedInitList is not None:
+ return bracedInitList
+ return self._parse_assignment_expression(inTemplate=False)
+
+ def _parse_braced_init_list(self) -> ASTBracedInitList:
+ # -> '{' initializer-list ','[opt] '}'
+ # | '{' '}'
+ exprs, trailingComma = self._parse_initializer_list("braced-init-list", '{', '}')
+ if exprs is None:
+ return None
+ return ASTBracedInitList(exprs, trailingComma)
+
+ def _parse_expression_list_or_braced_init_list(
+ self,
+ ) -> ASTParenExprList | ASTBracedInitList:
+ paren = self._parse_paren_expression_list()
+ if paren is not None:
+ return paren
+ return self._parse_braced_init_list()
+
+ def _parse_postfix_expression(self) -> ASTPostfixExpr:
+ # -> primary
+ # | postfix "[" expression "]"
+ # | postfix "[" braced-init-list [opt] "]"
+ # | postfix "(" expression-list [opt] ")"
+ # | postfix "." "template" [opt] id-expression
+ # | postfix "->" "template" [opt] id-expression
+ # | postfix "." pseudo-destructor-name
+ # | postfix "->" pseudo-destructor-name
+ # | postfix "++"
+ # | postfix "--"
+ # | simple-type-specifier "(" expression-list [opt] ")"
+ # | simple-type-specifier braced-init-list
+ # | typename-specifier "(" expression-list [opt] ")"
+ # | typename-specifier braced-init-list
+ # | "dynamic_cast" "<" type-id ">" "(" expression ")"
+ # | "static_cast" "<" type-id ">" "(" expression ")"
+ # | "reinterpret_cast" "<" type-id ">" "(" expression ")"
+ # | "const_cast" "<" type-id ">" "(" expression ")"
+ # | "typeid" "(" expression ")"
+ # | "typeid" "(" type-id ")"
+
+ prefixType = None
+ prefix: Any = None
+ self.skip_ws()
+
+ cast = None
+ for c in _id_explicit_cast:
+ if self.skip_word_and_ws(c):
+ cast = c
+ break
+ if cast is not None:
+ prefixType = "cast"
+ if not self.skip_string("<"):
+ self.fail("Expected '<' after '%s'." % cast)
+ typ = self._parse_type(False)
+ self.skip_ws()
+ if not self.skip_string_and_ws(">"):
+ self.fail("Expected '>' after type in '%s'." % cast)
+ if not self.skip_string("("):
+ self.fail("Expected '(' in '%s'." % cast)
+
+ def parser() -> ASTExpression:
+ return self._parse_expression()
+ expr = self._parse_expression_fallback([')'], parser)
+ self.skip_ws()
+ if not self.skip_string(")"):
+ self.fail("Expected ')' to end '%s'." % cast)
+ prefix = ASTExplicitCast(cast, typ, expr)
+ elif self.skip_word_and_ws("typeid"):
+ prefixType = "typeid"
+ if not self.skip_string_and_ws('('):
+ self.fail("Expected '(' after 'typeid'.")
+ pos = self.pos
+ try:
+ typ = self._parse_type(False)
+ prefix = ASTTypeId(typ, isType=True)
+ if not self.skip_string(')'):
+ self.fail("Expected ')' to end 'typeid' of type.")
+ except DefinitionError as eType:
+ self.pos = pos
+ try:
+
+ def parser() -> ASTExpression:
+ return self._parse_expression()
+ expr = self._parse_expression_fallback([')'], parser)
+ prefix = ASTTypeId(expr, isType=False)
+ if not self.skip_string(')'):
+ self.fail("Expected ')' to end 'typeid' of expression.")
+ except DefinitionError as eExpr:
+ self.pos = pos
+ header = "Error in 'typeid(...)'."
+ header += " Expected type or expression."
+ errors = []
+ errors.append((eType, "If type"))
+ errors.append((eExpr, "If expression"))
+ raise self._make_multi_error(errors, header) from eExpr
+ else: # a primary expression or a type
+ pos = self.pos
+ try:
+ prefix = self._parse_primary_expression()
+ prefixType = 'expr'
+ except DefinitionError as eOuter:
+ self.pos = pos
+ try:
+ # we are potentially casting, so save parens for us
+ # TODO: hmm, would we need to try both with operatorCast and with None?
+ prefix = self._parse_type(False, 'operatorCast')
+ prefixType = 'typeOperatorCast'
+ # | simple-type-specifier "(" expression-list [opt] ")"
+ # | simple-type-specifier braced-init-list
+ # | typename-specifier "(" expression-list [opt] ")"
+ # | typename-specifier braced-init-list
+ self.skip_ws()
+ if self.current_char != '(' and self.current_char != '{':
+ self.fail("Expecting '(' or '{' after type in cast expression.")
+ except DefinitionError as eInner:
+ self.pos = pos
+ header = "Error in postfix expression,"
+ header += " expected primary expression or type."
+ errors = []
+ errors.append((eOuter, "If primary expression"))
+ errors.append((eInner, "If type"))
+ raise self._make_multi_error(errors, header) from eInner
+
+ # and now parse postfixes
+ postFixes: list[ASTPostfixOp] = []
+ while True:
+ self.skip_ws()
+ if prefixType in ('expr', 'cast', 'typeid'):
+ if self.skip_string_and_ws('['):
+ expr = self._parse_expression()
+ self.skip_ws()
+ if not self.skip_string(']'):
+ self.fail("Expected ']' in end of postfix expression.")
+ postFixes.append(ASTPostfixArray(expr))
+ continue
+ if self.skip_string('.'):
+ if self.skip_string('*'):
+ # don't steal the dot
+ self.pos -= 2
+ elif self.skip_string('..'):
+ # don't steal the dot
+ self.pos -= 3
+ else:
+ name = self._parse_nested_name()
+ postFixes.append(ASTPostfixMember(name))
+ continue
+ if self.skip_string('->'):
+ if self.skip_string('*'):
+ # don't steal the arrow
+ self.pos -= 3
+ else:
+ name = self._parse_nested_name()
+ postFixes.append(ASTPostfixMemberOfPointer(name))
+ continue
+ if self.skip_string('++'):
+ postFixes.append(ASTPostfixInc())
+ continue
+ if self.skip_string('--'):
+ postFixes.append(ASTPostfixDec())
+ continue
+ lst = self._parse_expression_list_or_braced_init_list()
+ if lst is not None:
+ postFixes.append(ASTPostfixCallExpr(lst))
+ continue
+ break
+ return ASTPostfixExpr(prefix, postFixes)
+
+ def _parse_unary_expression(self) -> ASTExpression:
+ # -> postfix
+ # | "++" cast
+ # | "--" cast
+ # | unary-operator cast -> (* | & | + | - | ! | ~) cast
+ # The rest:
+ # | "sizeof" unary
+ # | "sizeof" "(" type-id ")"
+ # | "sizeof" "..." "(" identifier ")"
+ # | "alignof" "(" type-id ")"
+ # | noexcept-expression -> noexcept "(" expression ")"
+ # | new-expression
+ # | delete-expression
+ self.skip_ws()
+ for op in _expression_unary_ops:
+ # TODO: hmm, should we be able to backtrack here?
+ if op[0] in 'cn':
+ res = self.skip_word(op)
+ else:
+ res = self.skip_string(op)
+ if res:
+ expr = self._parse_cast_expression()
+ return ASTUnaryOpExpr(op, expr)
+ if self.skip_word_and_ws('sizeof'):
+ if self.skip_string_and_ws('...'):
+ if not self.skip_string_and_ws('('):
+ self.fail("Expecting '(' after 'sizeof...'.")
+ if not self.match(identifier_re):
+ self.fail("Expecting identifier for 'sizeof...'.")
+ ident = ASTIdentifier(self.matched_text)
+ self.skip_ws()
+ if not self.skip_string(")"):
+ self.fail("Expecting ')' to end 'sizeof...'.")
+ return ASTSizeofParamPack(ident)
+ if self.skip_string_and_ws('('):
+ typ = self._parse_type(named=False)
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expecting ')' to end 'sizeof'.")
+ return ASTSizeofType(typ)
+ expr = self._parse_unary_expression()
+ return ASTSizeofExpr(expr)
+ if self.skip_word_and_ws('alignof'):
+ if not self.skip_string_and_ws('('):
+ self.fail("Expecting '(' after 'alignof'.")
+ typ = self._parse_type(named=False)
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expecting ')' to end 'alignof'.")
+ return ASTAlignofExpr(typ)
+ if self.skip_word_and_ws('noexcept'):
+ if not self.skip_string_and_ws('('):
+ self.fail("Expecting '(' after 'noexcept'.")
+ expr = self._parse_expression()
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expecting ')' to end 'noexcept'.")
+ return ASTNoexceptExpr(expr)
+ # new-expression
+ pos = self.pos
+ rooted = self.skip_string('::')
+ self.skip_ws()
+ if not self.skip_word_and_ws('new'):
+ self.pos = pos
+ else:
+ # new-placement[opt] new-type-id new-initializer[opt]
+ # new-placement[opt] ( type-id ) new-initializer[opt]
+ isNewTypeId = True
+ if self.skip_string_and_ws('('):
+ # either this is a new-placement or it's the second production
+ # without placement, and it's actually the ( type-id ) part
+ self.fail("Sorry, neither new-placement nor parenthesised type-id "
+ "in new-epression is supported yet.")
+ # set isNewTypeId = False if it's (type-id)
+ if isNewTypeId:
+ declSpecs = self._parse_decl_specs(outer=None)
+ decl = self._parse_declarator(named=False, paramMode="new")
+ else:
+ self.fail("Sorry, parenthesised type-id in new expression not yet supported.")
+ lst = self._parse_expression_list_or_braced_init_list()
+ return ASTNewExpr(rooted, isNewTypeId, ASTType(declSpecs, decl), lst)
+ # delete-expression
+ pos = self.pos
+ rooted = self.skip_string('::')
+ self.skip_ws()
+ if not self.skip_word_and_ws('delete'):
+ self.pos = pos
+ else:
+ array = self.skip_string_and_ws('[')
+ if array and not self.skip_string_and_ws(']'):
+ self.fail("Expected ']' in array delete-expression.")
+ expr = self._parse_cast_expression()
+ return ASTDeleteExpr(rooted, array, expr)
+ return self._parse_postfix_expression()
+
+ def _parse_cast_expression(self) -> ASTExpression:
+ # -> unary | "(" type-id ")" cast
+ pos = self.pos
+ self.skip_ws()
+ if self.skip_string('('):
+ try:
+ typ = self._parse_type(False)
+ if not self.skip_string(')'):
+ self.fail("Expected ')' in cast expression.")
+ expr = self._parse_cast_expression()
+ return ASTCastExpr(typ, expr)
+ except DefinitionError as exCast:
+ self.pos = pos
+ try:
+ return self._parse_unary_expression()
+ except DefinitionError as exUnary:
+ errs = []
+ errs.append((exCast, "If type cast expression"))
+ errs.append((exUnary, "If unary expression"))
+ raise self._make_multi_error(errs,
+ "Error in cast expression.") from exUnary
+ else:
+ return self._parse_unary_expression()
+
+ def _parse_logical_or_expression(self, inTemplate: bool) -> ASTExpression:
+ # logical-or = logical-and ||
+ # logical-and = inclusive-or &&
+ # inclusive-or = exclusive-or |
+ # exclusive-or = and ^
+ # and = equality &
+ # equality = relational ==, !=
+ # relational = shift <, >, <=, >=, <=>
+ # shift = additive <<, >>
+ # additive = multiplicative +, -
+ # multiplicative = pm *, /, %
+ # pm = cast .*, ->*
+ def _parse_bin_op_expr(self: DefinitionParser,
+ opId: int, inTemplate: bool) -> ASTExpression:
+ if opId + 1 == len(_expression_bin_ops):
+ def parser(inTemplate: bool) -> ASTExpression:
+ return self._parse_cast_expression()
+ else:
+ def parser(inTemplate: bool) -> ASTExpression:
+ return _parse_bin_op_expr(self, opId + 1, inTemplate=inTemplate)
+ exprs = []
+ ops = []
+ exprs.append(parser(inTemplate=inTemplate))
+ while True:
+ self.skip_ws()
+ if inTemplate and self.current_char == '>':
+ break
+ pos = self.pos
+ oneMore = False
+ for op in _expression_bin_ops[opId]:
+ if op[0] in 'abcnox':
+ if not self.skip_word(op):
+ continue
+ else:
+ if not self.skip_string(op):
+ continue
+ if op == self.current_char == '&':
+ # don't split the && 'token'
+ self.pos -= 1
+ # and btw. && has lower precedence, so we are done
+ break
+ try:
+ expr = parser(inTemplate=inTemplate)
+ exprs.append(expr)
+ ops.append(op)
+ oneMore = True
+ break
+ except DefinitionError:
+ self.pos = pos
+ if not oneMore:
+ break
+ return ASTBinOpExpr(exprs, ops)
+ return _parse_bin_op_expr(self, 0, inTemplate=inTemplate)
+
+ def _parse_conditional_expression_tail(self, orExprHead: ASTExpression,
+ inTemplate: bool) -> ASTConditionalExpr | None:
+ # Consumes the orExprHead on success.
+
+ # -> "?" expression ":" assignment-expression
+ self.skip_ws()
+ if not self.skip_string("?"):
+ return None
+ thenExpr = self._parse_expression()
+ self.skip_ws()
+ if not self.skip_string(":"):
+ self.fail('Expected ":" after then-expression in conditional expression.')
+ elseExpr = self._parse_assignment_expression(inTemplate)
+ return ASTConditionalExpr(orExprHead, thenExpr, elseExpr)
+
+ def _parse_assignment_expression(self, inTemplate: bool) -> ASTExpression:
+ # -> conditional-expression
+ # | logical-or-expression assignment-operator initializer-clause
+ # | yield-expression -> "co_yield" assignment-expression
+ # | "co_yield" braced-init-list
+ # | throw-expression -> "throw" assignment-expression[opt]
+ # TODO: yield-expression
+ # TODO: throw-expression
+
+ # Now we have (after expanding conditional-expression:
+ # logical-or-expression
+ # | logical-or-expression "?" expression ":" assignment-expression
+ # | logical-or-expression assignment-operator initializer-clause
+ leftExpr = self._parse_logical_or_expression(inTemplate=inTemplate)
+ # the ternary operator
+ condExpr = self._parse_conditional_expression_tail(leftExpr, inTemplate)
+ if condExpr is not None:
+ return condExpr
+ # and actual assignment
+ for op in _expression_assignment_ops:
+ if op[0] in 'anox':
+ if not self.skip_word(op):
+ continue
+ else:
+ if not self.skip_string(op):
+ continue
+ rightExpr = self._parse_initializer_clause()
+ return ASTAssignmentExpr(leftExpr, op, rightExpr)
+ # just a logical-or-expression
+ return leftExpr
+
+ def _parse_constant_expression(self, inTemplate: bool) -> ASTExpression:
+ # -> conditional-expression ->
+ # logical-or-expression
+ # | logical-or-expression "?" expression ":" assignment-expression
+ orExpr = self._parse_logical_or_expression(inTemplate=inTemplate)
+ condExpr = self._parse_conditional_expression_tail(orExpr, inTemplate)
+ if condExpr is not None:
+ return condExpr
+ return orExpr
+
+ def _parse_expression(self) -> ASTExpression:
+ # -> assignment-expression
+ # | expression "," assignment-expression
+ exprs = [self._parse_assignment_expression(inTemplate=False)]
+ while True:
+ self.skip_ws()
+ if not self.skip_string(','):
+ break
+ exprs.append(self._parse_assignment_expression(inTemplate=False))
+ if len(exprs) == 1:
+ return exprs[0]
+ else:
+ return ASTCommaExpr(exprs)
+
+ def _parse_expression_fallback(self, end: list[str],
+ parser: Callable[[], ASTExpression],
+ allow: bool = True) -> ASTExpression:
+ # Stupidly "parse" an expression.
+ # 'end' should be a list of characters which ends the expression.
+
+ # first try to use the provided parser
+ prevPos = self.pos
+ try:
+ return parser()
+ except DefinitionError as e:
+ # some places (e.g., template parameters) we really don't want to use fallback,
+ # and for testing we may want to globally disable it
+ if not allow or not self.allowFallbackExpressionParsing:
+ raise
+ self.warn("Parsing of expression failed. Using fallback parser."
+ " Error was:\n%s" % e)
+ self.pos = prevPos
+ # and then the fallback scanning
+ assert end is not None
+ self.skip_ws()
+ startPos = self.pos
+ if self.match(_string_re):
+ value = self.matched_text
+ else:
+ # TODO: add handling of more bracket-like things, and quote handling
+ brackets = {'(': ')', '{': '}', '[': ']', '<': '>'}
+ symbols: list[str] = []
+ while not self.eof:
+ if (len(symbols) == 0 and self.current_char in end):
+ break
+ if self.current_char in brackets:
+ symbols.append(brackets[self.current_char])
+ elif len(symbols) > 0 and self.current_char == symbols[-1]:
+ symbols.pop()
+ self.pos += 1
+ if len(end) > 0 and self.eof:
+ self.fail("Could not find end of expression starting at %d."
+ % startPos)
+ value = self.definition[startPos:self.pos].strip()
+ return ASTFallbackExpr(value.strip())
+
+ # ==========================================================================
+
+ def _parse_operator(self) -> ASTOperator:
+ self.skip_ws()
+ # adapted from the old code
+ # yay, a regular operator definition
+ if self.match(_operator_re):
+ return ASTOperatorBuildIn(self.matched_text)
+
+ # new/delete operator?
+ for op in 'new', 'delete':
+ if not self.skip_word(op):
+ continue
+ self.skip_ws()
+ if self.skip_string('['):
+ self.skip_ws()
+ if not self.skip_string(']'):
+ self.fail('Expected "]" after "operator ' + op + '["')
+ op += '[]'
+ return ASTOperatorBuildIn(op)
+
+ # user-defined literal?
+ if self.skip_string('""'):
+ self.skip_ws()
+ if not self.match(identifier_re):
+ self.fail("Expected user-defined literal suffix.")
+ identifier = ASTIdentifier(self.matched_text)
+ return ASTOperatorLiteral(identifier)
+
+ # oh well, looks like a cast operator definition.
+ # In that case, eat another type.
+ type = self._parse_type(named=False, outer="operatorCast")
+ return ASTOperatorType(type)
+
+ def _parse_template_argument_list(self) -> ASTTemplateArgs:
+ # template-argument-list: (but we include the < and > here
+ # template-argument ...[opt]
+ # template-argument-list, template-argument ...[opt]
+ # template-argument:
+ # constant-expression
+ # type-id
+ # id-expression
+ self.skip_ws()
+ if not self.skip_string_and_ws('<'):
+ return None
+ if self.skip_string('>'):
+ return ASTTemplateArgs([], False)
+ prevErrors = []
+ templateArgs: list[ASTType | ASTTemplateArgConstant] = []
+ packExpansion = False
+ while 1:
+ pos = self.pos
+ parsedComma = False
+ parsedEnd = False
+ try:
+ type = self._parse_type(named=False)
+ self.skip_ws()
+ if self.skip_string_and_ws('...'):
+ packExpansion = True
+ parsedEnd = True
+ if not self.skip_string('>'):
+ self.fail('Expected ">" after "..." in template argument list.')
+ elif self.skip_string('>'):
+ parsedEnd = True
+ elif self.skip_string(','):
+ parsedComma = True
+ else:
+ self.fail('Expected "...>", ">" or "," in template argument list.')
+ templateArgs.append(type)
+ except DefinitionError as e:
+ prevErrors.append((e, "If type argument"))
+ self.pos = pos
+ try:
+ value = self._parse_constant_expression(inTemplate=True)
+ self.skip_ws()
+ if self.skip_string_and_ws('...'):
+ packExpansion = True
+ parsedEnd = True
+ if not self.skip_string('>'):
+ self.fail('Expected ">" after "..." in template argument list.')
+ elif self.skip_string('>'):
+ parsedEnd = True
+ elif self.skip_string(','):
+ parsedComma = True
+ else:
+ self.fail('Expected "...>", ">" or "," in template argument list.')
+ templateArgs.append(ASTTemplateArgConstant(value))
+ except DefinitionError as e:
+ self.pos = pos
+ prevErrors.append((e, "If non-type argument"))
+ header = "Error in parsing template argument list."
+ raise self._make_multi_error(prevErrors, header) from e
+ if parsedEnd:
+ assert not parsedComma
+ break
+ assert not packExpansion
+ return ASTTemplateArgs(templateArgs, packExpansion)
+
+ def _parse_nested_name(self, memberPointer: bool = False) -> ASTNestedName:
+ names: list[ASTNestedNameElement] = []
+ templates: list[bool] = []
+
+ self.skip_ws()
+ rooted = False
+ if self.skip_string('::'):
+ rooted = True
+ while 1:
+ self.skip_ws()
+ if len(names) > 0:
+ template = self.skip_word_and_ws('template')
+ else:
+ template = False
+ templates.append(template)
+ identOrOp: ASTIdentifier | ASTOperator | None = None
+ if self.skip_word_and_ws('operator'):
+ identOrOp = self._parse_operator()
+ else:
+ if not self.match(identifier_re):
+ if memberPointer and len(names) > 0:
+ templates.pop()
+ break
+ self.fail("Expected identifier in nested name.")
+ identifier = self.matched_text
+ # make sure there isn't a keyword
+ if identifier in _keywords:
+ self.fail("Expected identifier in nested name, "
+ "got keyword: %s" % identifier)
+ identOrOp = ASTIdentifier(identifier)
+ # try greedily to get template arguments,
+ # but otherwise a < might be because we are in an expression
+ pos = self.pos
+ try:
+ templateArgs = self._parse_template_argument_list()
+ except DefinitionError as ex:
+ self.pos = pos
+ templateArgs = None
+ self.otherErrors.append(ex)
+ names.append(ASTNestedNameElement(identOrOp, templateArgs))
+
+ self.skip_ws()
+ if not self.skip_string('::'):
+ if memberPointer:
+ self.fail("Expected '::' in pointer to member (function).")
+ break
+ return ASTNestedName(names, templates, rooted)
+
+ # ==========================================================================
+
+ def _parse_simple_type_specifiers(self) -> ASTTrailingTypeSpecFundamental:
+ modifier: str | None = None
+ signedness: str | None = None
+ width: list[str] = []
+ typ: str | None = None
+ names: list[str] = [] # the parsed sequence
+
+ self.skip_ws()
+ while self.match(_simple_type_specifiers_re):
+ t = self.matched_text
+ names.append(t)
+ if t in ('auto', 'void', 'bool',
+ 'char', 'wchar_t', 'char8_t', 'char16_t', 'char32_t',
+ 'int', '__int64', '__int128',
+ 'float', 'double',
+ '__float80', '_Float64x', '__float128', '_Float128'):
+ if typ is not None:
+ self.fail(f"Can not have both {t} and {typ}.")
+ typ = t
+ elif t in ('signed', 'unsigned'):
+ if signedness is not None:
+ self.fail(f"Can not have both {t} and {signedness}.")
+ signedness = t
+ elif t == 'short':
+ if len(width) != 0:
+ self.fail(f"Can not have both {t} and {width[0]}.")
+ width.append(t)
+ elif t == 'long':
+ if len(width) != 0 and width[0] != 'long':
+ self.fail(f"Can not have both {t} and {width[0]}.")
+ width.append(t)
+ elif t in ('_Imaginary', '_Complex'):
+ if modifier is not None:
+ self.fail(f"Can not have both {t} and {modifier}.")
+ modifier = t
+ self.skip_ws()
+ if len(names) == 0:
+ return None
+
+ if typ in ('auto', 'void', 'bool',
+ 'wchar_t', 'char8_t', 'char16_t', 'char32_t',
+ '__float80', '_Float64x', '__float128', '_Float128'):
+ if modifier is not None:
+ self.fail(f"Can not have both {typ} and {modifier}.")
+ if signedness is not None:
+ self.fail(f"Can not have both {typ} and {signedness}.")
+ if len(width) != 0:
+ self.fail(f"Can not have both {typ} and {' '.join(width)}.")
+ elif typ == 'char':
+ if modifier is not None:
+ self.fail(f"Can not have both {typ} and {modifier}.")
+ if len(width) != 0:
+ self.fail(f"Can not have both {typ} and {' '.join(width)}.")
+ elif typ == 'int':
+ if modifier is not None:
+ self.fail(f"Can not have both {typ} and {modifier}.")
+ elif typ in ('__int64', '__int128'):
+ if modifier is not None:
+ self.fail(f"Can not have both {typ} and {modifier}.")
+ if len(width) != 0:
+ self.fail(f"Can not have both {typ} and {' '.join(width)}.")
+ elif typ == 'float':
+ if signedness is not None:
+ self.fail(f"Can not have both {typ} and {signedness}.")
+ if len(width) != 0:
+ self.fail(f"Can not have both {typ} and {' '.join(width)}.")
+ elif typ == 'double':
+ if signedness is not None:
+ self.fail(f"Can not have both {typ} and {signedness}.")
+ if len(width) > 1:
+ self.fail(f"Can not have both {typ} and {' '.join(width)}.")
+ if len(width) == 1 and width[0] != 'long':
+ self.fail(f"Can not have both {typ} and {' '.join(width)}.")
+ elif typ is None:
+ if modifier is not None:
+ self.fail(f"Can not have {modifier} without a floating point type.")
+ else:
+ msg = f'Unhandled type {typ}'
+ raise AssertionError(msg)
+
+ canonNames: list[str] = []
+ if modifier is not None:
+ canonNames.append(modifier)
+ if signedness is not None:
+ canonNames.append(signedness)
+ canonNames.extend(width)
+ if typ is not None:
+ canonNames.append(typ)
+ return ASTTrailingTypeSpecFundamental(names, canonNames)
+
+ def _parse_trailing_type_spec(self) -> ASTTrailingTypeSpec:
+ # fundamental types, https://en.cppreference.com/w/cpp/language/type
+ # and extensions
+ self.skip_ws()
+ res = self._parse_simple_type_specifiers()
+ if res is not None:
+ return res
+
+ # decltype
+ self.skip_ws()
+ if self.skip_word_and_ws('decltype'):
+ if not self.skip_string_and_ws('('):
+ self.fail("Expected '(' after 'decltype'.")
+ if self.skip_word_and_ws('auto'):
+ if not self.skip_string(')'):
+ self.fail("Expected ')' after 'decltype(auto'.")
+ return ASTTrailingTypeSpecDecltypeAuto()
+ expr = self._parse_expression()
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expected ')' after 'decltype(<expr>'.")
+ return ASTTrailingTypeSpecDecltype(expr)
+
+ # prefixed
+ prefix = None
+ self.skip_ws()
+ for k in ('class', 'struct', 'enum', 'union', 'typename'):
+ if self.skip_word_and_ws(k):
+ prefix = k
+ break
+ nestedName = self._parse_nested_name()
+ self.skip_ws()
+ placeholderType = None
+ if self.skip_word('auto'):
+ placeholderType = 'auto'
+ elif self.skip_word_and_ws('decltype'):
+ if not self.skip_string_and_ws('('):
+ self.fail("Expected '(' after 'decltype' in placeholder type specifier.")
+ if not self.skip_word_and_ws('auto'):
+ self.fail("Expected 'auto' after 'decltype(' in placeholder type specifier.")
+ if not self.skip_string_and_ws(')'):
+ self.fail("Expected ')' after 'decltype(auto' in placeholder type specifier.")
+ placeholderType = 'decltype(auto)'
+ return ASTTrailingTypeSpecName(prefix, nestedName, placeholderType)
+
+ def _parse_parameters_and_qualifiers(
+ self, paramMode: str,
+ ) -> ASTParametersQualifiers | None:
+ if paramMode == 'new':
+ return None
+ self.skip_ws()
+ if not self.skip_string('('):
+ if paramMode == 'function':
+ self.fail('Expecting "(" in parameters-and-qualifiers.')
+ else:
+ return None
+ args = []
+ self.skip_ws()
+ if not self.skip_string(')'):
+ while 1:
+ self.skip_ws()
+ if self.skip_string('...'):
+ args.append(ASTFunctionParameter(None, True))
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail('Expected ")" after "..." in '
+ 'parameters-and-qualifiers.')
+ break
+ # note: it seems that function arguments can always be named,
+ # even in function pointers and similar.
+ arg = self._parse_type_with_init(outer=None, named='single')
+ # TODO: parse default parameters # TODO: didn't we just do that?
+ args.append(ASTFunctionParameter(arg))
- def _parse_decl_specs_simple(self, outer: str, typed: bool
- ) ->ASTDeclSpecsSimple:
+ self.skip_ws()
+ if self.skip_string(','):
+ continue
+ if self.skip_string(')'):
+ break
+ self.fail('Expecting "," or ")" in parameters-and-qualifiers, '
+ f'got "{self.current_char}".')
+
+ self.skip_ws()
+ const = self.skip_word_and_ws('const')
+ volatile = self.skip_word_and_ws('volatile')
+ if not const: # the can be permuted
+ const = self.skip_word_and_ws('const')
+
+ refQual = None
+ if self.skip_string('&&'):
+ refQual = '&&'
+ if not refQual and self.skip_string('&'):
+ refQual = '&'
+
+ exceptionSpec = None
+ self.skip_ws()
+ if self.skip_string('noexcept'):
+ if self.skip_string_and_ws('('):
+ expr = self._parse_constant_expression(False)
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expecting ')' to end 'noexcept'.")
+ exceptionSpec = ASTNoexceptSpec(expr)
+ else:
+ exceptionSpec = ASTNoexceptSpec(None)
+
+ self.skip_ws()
+ if self.skip_string('->'):
+ trailingReturn = self._parse_type(named=False)
+ else:
+ trailingReturn = None
+
+ self.skip_ws()
+ override = self.skip_word_and_ws('override')
+ final = self.skip_word_and_ws('final')
+ if not override:
+ override = self.skip_word_and_ws(
+ 'override') # they can be permuted
+
+ attrs = self._parse_attribute_list()
+
+ self.skip_ws()
+ initializer = None
+ # if this is a function pointer we should not swallow an initializer
+ if paramMode == 'function' and self.skip_string('='):
+ self.skip_ws()
+ valid = ('0', 'delete', 'default')
+ for w in valid:
+ if self.skip_word_and_ws(w):
+ initializer = w
+ break
+ if not initializer:
+ self.fail(
+ 'Expected "%s" in initializer-specifier.'
+ % '" or "'.join(valid))
+
+ return ASTParametersQualifiers(
+ args, volatile, const, refQual, exceptionSpec, trailingReturn,
+ override, final, attrs, initializer)
+
+ def _parse_decl_specs_simple(self, outer: str, typed: bool) -> ASTDeclSpecsSimple:
"""Just parse the simple ones."""
- pass
+ storage = None
+ threadLocal = None
+ inline = None
+ virtual = None
+ explicitSpec = None
+ consteval = None
+ constexpr = None
+ constinit = None
+ volatile = None
+ const = None
+ friend = None
+ attrs = []
+ while 1: # accept any permutation of a subset of some decl-specs
+ self.skip_ws()
+ if not const and typed:
+ const = self.skip_word('const')
+ if const:
+ continue
+ if not volatile and typed:
+ volatile = self.skip_word('volatile')
+ if volatile:
+ continue
+ if not storage:
+ if outer in ('member', 'function'):
+ if self.skip_word('static'):
+ storage = 'static'
+ continue
+ if self.skip_word('extern'):
+ storage = 'extern'
+ continue
+ if outer == 'member':
+ if self.skip_word('mutable'):
+ storage = 'mutable'
+ continue
+ if self.skip_word('register'):
+ storage = 'register'
+ continue
+ if not inline and outer in ('function', 'member'):
+ inline = self.skip_word('inline')
+ if inline:
+ continue
+ if not constexpr and outer in ('member', 'function'):
+ constexpr = self.skip_word("constexpr")
+ if constexpr:
+ continue
+
+ if outer == 'member':
+ if not constinit:
+ constinit = self.skip_word('constinit')
+ if constinit:
+ continue
+ if not threadLocal:
+ threadLocal = self.skip_word('thread_local')
+ if threadLocal:
+ continue
+ if outer == 'function':
+ if not consteval:
+ consteval = self.skip_word('consteval')
+ if consteval:
+ continue
+ if not friend:
+ friend = self.skip_word('friend')
+ if friend:
+ continue
+ if not virtual:
+ virtual = self.skip_word('virtual')
+ if virtual:
+ continue
+ if not explicitSpec:
+ explicit = self.skip_word_and_ws('explicit')
+ if explicit:
+ expr: ASTExpression = None
+ if self.skip_string('('):
+ expr = self._parse_constant_expression(inTemplate=False)
+ if not expr:
+ self.fail("Expected constant expression after '('"
+ " in explicit specifier.")
+ self.skip_ws()
+ if not self.skip_string(')'):
+ self.fail("Expected ')' to end explicit specifier.")
+ explicitSpec = ASTExplicitSpec(expr)
+ continue
+ attr = self._parse_attribute()
+ if attr:
+ attrs.append(attr)
+ continue
+ break
+ return ASTDeclSpecsSimple(storage, threadLocal, inline, virtual,
+ explicitSpec, consteval, constexpr, constinit,
+ volatile, const, friend, ASTAttributeList(attrs))
+
+ def _parse_decl_specs(self, outer: str, typed: bool = True) -> ASTDeclSpecs:
+ if outer:
+ if outer not in ('type', 'member', 'function', 'templateParam'):
+ raise Exception('Internal error, unknown outer "%s".' % outer)
+ """
+ storage-class-specifier function-specifier "constexpr"
+ "volatile" "const" trailing-type-specifier
+
+ storage-class-specifier ->
+ "static" (only for member_object and function_object)
+ | "register"
+
+ function-specifier -> "inline" | "virtual" | "explicit" (only for
+ function_object)
- def _parse_type(self, named: (bool | str), outer: (str | None)=None
- ) ->ASTType:
+ "constexpr" (only for member_object and function_object)
+ """
+ leftSpecs = self._parse_decl_specs_simple(outer, typed)
+ rightSpecs = None
+
+ if typed:
+ trailing = self._parse_trailing_type_spec()
+ rightSpecs = self._parse_decl_specs_simple(outer, typed)
+ else:
+ trailing = None
+ return ASTDeclSpecs(outer, leftSpecs, rightSpecs, trailing)
+
+ def _parse_declarator_name_suffix(
+ self, named: bool | str, paramMode: str, typed: bool,
+ ) -> ASTDeclaratorNameParamQual | ASTDeclaratorNameBitField:
+ # now we should parse the name, and then suffixes
+ if named == 'maybe':
+ pos = self.pos
+ try:
+ declId = self._parse_nested_name()
+ except DefinitionError:
+ self.pos = pos
+ declId = None
+ elif named == 'single':
+ if self.match(identifier_re):
+ identifier = ASTIdentifier(self.matched_text)
+ nne = ASTNestedNameElement(identifier, None)
+ declId = ASTNestedName([nne], [False], rooted=False)
+ # if it's a member pointer, we may have '::', which should be an error
+ self.skip_ws()
+ if self.current_char == ':':
+ self.fail("Unexpected ':' after identifier.")
+ else:
+ declId = None
+ elif named:
+ declId = self._parse_nested_name()
+ else:
+ declId = None
+ arrayOps = []
+ while 1:
+ self.skip_ws()
+ if typed and self.skip_string('['):
+ self.skip_ws()
+ if self.skip_string(']'):
+ arrayOps.append(ASTArray(None))
+ continue
+
+ def parser() -> ASTExpression:
+ return self._parse_expression()
+ value = self._parse_expression_fallback([']'], parser)
+ if not self.skip_string(']'):
+ self.fail("Expected ']' in end of array operator.")
+ arrayOps.append(ASTArray(value))
+ continue
+ break
+ paramQual = self._parse_parameters_and_qualifiers(paramMode)
+ if paramQual is None and len(arrayOps) == 0:
+ # perhaps a bit-field
+ if named and paramMode == 'type' and typed:
+ self.skip_ws()
+ if self.skip_string(':'):
+ size = self._parse_constant_expression(inTemplate=False)
+ return ASTDeclaratorNameBitField(declId=declId, size=size)
+ return ASTDeclaratorNameParamQual(declId=declId, arrayOps=arrayOps,
+ paramQual=paramQual)
+
+ def _parse_declarator(self, named: bool | str, paramMode: str,
+ typed: bool = True,
+ ) -> ASTDeclarator:
+ # 'typed' here means 'parse return type stuff'
+ if paramMode not in ('type', 'function', 'operatorCast', 'new'):
+ raise Exception(
+ "Internal error, unknown paramMode '%s'." % paramMode)
+ prevErrors = []
+ self.skip_ws()
+ if typed and self.skip_string('*'):
+ self.skip_ws()
+ volatile = False
+ const = False
+ attrList = []
+ while 1:
+ if not volatile:
+ volatile = self.skip_word_and_ws('volatile')
+ if volatile:
+ continue
+ if not const:
+ const = self.skip_word_and_ws('const')
+ if const:
+ continue
+ attr = self._parse_attribute()
+ if attr is not None:
+ attrList.append(attr)
+ continue
+ break
+ next = self._parse_declarator(named, paramMode, typed)
+ return ASTDeclaratorPtr(next=next, volatile=volatile, const=const,
+ attrs=ASTAttributeList(attrList))
+ # TODO: shouldn't we parse an R-value ref here first?
+ if typed and self.skip_string("&"):
+ attrs = self._parse_attribute_list()
+ next = self._parse_declarator(named, paramMode, typed)
+ return ASTDeclaratorRef(next=next, attrs=attrs)
+ if typed and self.skip_string("..."):
+ next = self._parse_declarator(named, paramMode, False)
+ return ASTDeclaratorParamPack(next=next)
+ if typed and self.current_char == '(': # note: peeking, not skipping
+ if paramMode == "operatorCast":
+ # TODO: we should be able to parse cast operators which return
+ # function pointers. For now, just hax it and ignore.
+ return ASTDeclaratorNameParamQual(declId=None, arrayOps=[],
+ paramQual=None)
+ # maybe this is the beginning of params and quals,try that first,
+ # otherwise assume it's noptr->declarator > ( ptr-declarator )
+ pos = self.pos
+ try:
+ # assume this is params and quals
+ res = self._parse_declarator_name_suffix(named, paramMode,
+ typed)
+ return res
+ except DefinitionError as exParamQual:
+ prevErrors.append((exParamQual,
+ "If declarator-id with parameters-and-qualifiers"))
+ self.pos = pos
+ try:
+ assert self.current_char == '('
+ self.skip_string('(')
+ # TODO: hmm, if there is a name, it must be in inner, right?
+ # TODO: hmm, if there must be parameters, they must be
+ # inside, right?
+ inner = self._parse_declarator(named, paramMode, typed)
+ if not self.skip_string(')'):
+ self.fail("Expected ')' in \"( ptr-declarator )\"")
+ next = self._parse_declarator(named=False,
+ paramMode="type",
+ typed=typed)
+ return ASTDeclaratorParen(inner=inner, next=next)
+ except DefinitionError as exNoPtrParen:
+ self.pos = pos
+ prevErrors.append((exNoPtrParen, "If parenthesis in noptr-declarator"))
+ header = "Error in declarator"
+ raise self._make_multi_error(prevErrors, header) from exNoPtrParen
+ if typed: # pointer to member
+ pos = self.pos
+ try:
+ name = self._parse_nested_name(memberPointer=True)
+ self.skip_ws()
+ if not self.skip_string('*'):
+ self.fail("Expected '*' in pointer to member declarator.")
+ self.skip_ws()
+ except DefinitionError as e:
+ self.pos = pos
+ prevErrors.append((e, "If pointer to member declarator"))
+ else:
+ volatile = False
+ const = False
+ while 1:
+ if not volatile:
+ volatile = self.skip_word_and_ws('volatile')
+ if volatile:
+ continue
+ if not const:
+ const = self.skip_word_and_ws('const')
+ if const:
+ continue
+ break
+ next = self._parse_declarator(named, paramMode, typed)
+ return ASTDeclaratorMemPtr(name, const, volatile, next=next)
+ pos = self.pos
+ try:
+ res = self._parse_declarator_name_suffix(named, paramMode, typed)
+ # this is a heuristic for error messages, for when there is a < after a
+ # nested name, but it was not a successful template argument list
+ if self.current_char == '<':
+ self.otherErrors.append(self._make_multi_error(prevErrors, ""))
+ return res
+ except DefinitionError as e:
+ self.pos = pos
+ prevErrors.append((e, "If declarator-id"))
+ header = "Error in declarator or parameters-and-qualifiers"
+ raise self._make_multi_error(prevErrors, header) from e
+
+ def _parse_initializer(self, outer: str | None = None, allowFallback: bool = True,
+ ) -> ASTInitializer | None:
+ # initializer # global vars
+ # -> brace-or-equal-initializer
+ # | '(' expression-list ')'
+ #
+ # brace-or-equal-initializer # member vars
+ # -> '=' initializer-clause
+ # | braced-init-list
+ #
+ # initializer-clause # function params, non-type template params (with '=' in front)
+ # -> assignment-expression
+ # | braced-init-list
+ #
+ # we don't distinguish between global and member vars, so disallow paren:
+ #
+ # -> braced-init-list # var only
+ # | '=' assignment-expression
+ # | '=' braced-init-list
+ self.skip_ws()
+ if outer == 'member':
+ bracedInit = self._parse_braced_init_list()
+ if bracedInit is not None:
+ return ASTInitializer(bracedInit, hasAssign=False)
+
+ if not self.skip_string('='):
+ return None
+
+ bracedInit = self._parse_braced_init_list()
+ if bracedInit is not None:
+ return ASTInitializer(bracedInit)
+
+ if outer == 'member':
+ fallbackEnd: list[str] = []
+ elif outer == 'templateParam':
+ fallbackEnd = [',', '>']
+ elif outer is None: # function parameter
+ fallbackEnd = [',', ')']
+ else:
+ self.fail("Internal error, initializer for outer '%s' not "
+ "implemented." % outer)
+
+ inTemplate = outer == 'templateParam'
+
+ def parser() -> ASTExpression:
+ return self._parse_assignment_expression(inTemplate=inTemplate)
+ value = self._parse_expression_fallback(fallbackEnd, parser, allow=allowFallback)
+ return ASTInitializer(value)
+
+ def _parse_type(self, named: bool | str, outer: str | None = None) -> ASTType:
"""
named=False|'maybe'|True: 'maybe' is e.g., for function objects which
doesn't need to name the arguments
outer == operatorCast: annoying case, we should not take the params
"""
- pass
+ if outer: # always named
+ if outer not in ('type', 'member', 'function',
+ 'operatorCast', 'templateParam'):
+ raise Exception('Internal error, unknown outer "%s".' % outer)
+ if outer != 'operatorCast':
+ assert named
+ if outer in ('type', 'function'):
+ # We allow type objects to just be a name.
+ # Some functions don't have normal return types: constructors,
+ # destructors, cast operators
+ prevErrors = []
+ startPos = self.pos
+ # first try without the type
+ try:
+ declSpecs = self._parse_decl_specs(outer=outer, typed=False)
+ decl = self._parse_declarator(named=True, paramMode=outer,
+ typed=False)
+ mustEnd = True
+ if outer == 'function':
+ # Allow trailing requires on functions.
+ self.skip_ws()
+ if re.compile(r'requires\b').match(self.definition, self.pos):
+ mustEnd = False
+ if mustEnd:
+ self.assert_end(allowSemicolon=True)
+ except DefinitionError as exUntyped:
+ if outer == 'type':
+ desc = "If just a name"
+ elif outer == 'function':
+ desc = "If the function has no return type"
+ else:
+ raise AssertionError from exUntyped
+ prevErrors.append((exUntyped, desc))
+ self.pos = startPos
+ try:
+ declSpecs = self._parse_decl_specs(outer=outer)
+ decl = self._parse_declarator(named=True, paramMode=outer)
+ except DefinitionError as exTyped:
+ self.pos = startPos
+ if outer == 'type':
+ desc = "If typedef-like declaration"
+ elif outer == 'function':
+ desc = "If the function has a return type"
+ else:
+ raise AssertionError from exUntyped
+ prevErrors.append((exTyped, desc))
+ # Retain the else branch for easier debugging.
+ # TODO: it would be nice to save the previous stacktrace
+ # and output it here.
+ if True:
+ if outer == 'type':
+ header = "Type must be either just a name or a "
+ header += "typedef-like declaration."
+ elif outer == 'function':
+ header = "Error when parsing function declaration."
+ else:
+ raise AssertionError from exUntyped
+ raise self._make_multi_error(prevErrors, header) from exTyped
+ else: # NoQA: RET506
+ # For testing purposes.
+ # do it again to get the proper traceback (how do you
+ # reliably save a traceback when an exception is
+ # constructed?)
+ self.pos = startPos
+ typed = True
+ declSpecs = self._parse_decl_specs(outer=outer, typed=typed)
+ decl = self._parse_declarator(named=True, paramMode=outer,
+ typed=typed)
+ else:
+ paramMode = 'type'
+ if outer == 'member':
+ named = True
+ elif outer == 'operatorCast':
+ paramMode = 'operatorCast'
+ outer = None
+ elif outer == 'templateParam':
+ named = 'single'
+ declSpecs = self._parse_decl_specs(outer=outer)
+ decl = self._parse_declarator(named=named, paramMode=paramMode)
+ return ASTType(declSpecs, decl)
+
+ def _parse_type_with_init(
+ self, named: bool | str,
+ outer: str) -> ASTTypeWithInit | ASTTemplateParamConstrainedTypeWithInit:
+ if outer:
+ assert outer in ('type', 'member', 'function', 'templateParam')
+ type = self._parse_type(outer=outer, named=named)
+ if outer != 'templateParam':
+ init = self._parse_initializer(outer=outer)
+ return ASTTypeWithInit(type, init)
+ # it could also be a constrained type parameter, e.g., C T = int&
+ pos = self.pos
+ eExpr = None
+ try:
+ init = self._parse_initializer(outer=outer, allowFallback=False)
+ # note: init may be None if there is no =
+ if init is None:
+ return ASTTypeWithInit(type, None)
+ # we parsed an expression, so we must have a , or a >,
+ # otherwise the expression didn't get everything
+ self.skip_ws()
+ if self.current_char != ',' and self.current_char != '>':
+ # pretend it didn't happen
+ self.pos = pos
+ init = None
+ else:
+ # we assume that it was indeed an expression
+ return ASTTypeWithInit(type, init)
+ except DefinitionError as e:
+ self.pos = pos
+ eExpr = e
+ if not self.skip_string("="):
+ return ASTTypeWithInit(type, None)
+ try:
+ typeInit = self._parse_type(named=False, outer=None)
+ return ASTTemplateParamConstrainedTypeWithInit(type, typeInit)
+ except DefinitionError as eType:
+ if eExpr is None:
+ raise
+ errs = []
+ errs.append((eExpr, "If default template argument is an expression"))
+ errs.append((eType, "If default template argument is a type"))
+ msg = "Error in non-type template parameter"
+ msg += " or constrained template parameter."
+ raise self._make_multi_error(errs, msg) from eType
+
+ def _parse_type_using(self) -> ASTTypeUsing:
+ name = self._parse_nested_name()
+ self.skip_ws()
+ if not self.skip_string('='):
+ return ASTTypeUsing(name, None)
+ type = self._parse_type(False, None)
+ return ASTTypeUsing(name, type)
+
+ def _parse_concept(self) -> ASTConcept:
+ nestedName = self._parse_nested_name()
+ self.skip_ws()
+ initializer = self._parse_initializer('member')
+ return ASTConcept(nestedName, initializer)
+
+ def _parse_class(self) -> ASTClass:
+ attrs = self._parse_attribute_list()
+ name = self._parse_nested_name()
+ self.skip_ws()
+ final = self.skip_word_and_ws('final')
+ bases = []
+ self.skip_ws()
+ if self.skip_string(':'):
+ while 1:
+ self.skip_ws()
+ visibility = None
+ virtual = False
+ pack = False
+ if self.skip_word_and_ws('virtual'):
+ virtual = True
+ if self.match(_visibility_re):
+ visibility = self.matched_text
+ self.skip_ws()
+ if not virtual and self.skip_word_and_ws('virtual'):
+ virtual = True
+ baseName = self._parse_nested_name()
+ self.skip_ws()
+ pack = self.skip_string('...')
+ bases.append(ASTBaseClass(baseName, visibility, virtual, pack))
+ self.skip_ws()
+ if self.skip_string(','):
+ continue
+ break
+ return ASTClass(name, final, bases, attrs)
+
+ def _parse_union(self) -> ASTUnion:
+ attrs = self._parse_attribute_list()
+ name = self._parse_nested_name()
+ return ASTUnion(name, attrs)
+
+ def _parse_enum(self) -> ASTEnum:
+ scoped = None # is set by CPPEnumObject
+ attrs = self._parse_attribute_list()
+ name = self._parse_nested_name()
+ self.skip_ws()
+ underlyingType = None
+ if self.skip_string(':'):
+ underlyingType = self._parse_type(named=False)
+ return ASTEnum(name, scoped, underlyingType, attrs)
+
+ def _parse_enumerator(self) -> ASTEnumerator:
+ name = self._parse_nested_name()
+ attrs = self._parse_attribute_list()
+ self.skip_ws()
+ init = None
+ if self.skip_string('='):
+ self.skip_ws()
+
+ def parser() -> ASTExpression:
+ return self._parse_constant_expression(inTemplate=False)
+ initVal = self._parse_expression_fallback([], parser)
+ init = ASTInitializer(initVal)
+ return ASTEnumerator(name, init, attrs)
+
+ # ==========================================================================
+
+ def _parse_template_parameter(self) -> ASTTemplateParam:
+ self.skip_ws()
+ if self.skip_word('template'):
+ # declare a template template parameter
+ nestedParams = self._parse_template_parameter_list()
+ else:
+ nestedParams = None
+
+ pos = self.pos
+ try:
+ # Unconstrained type parameter or template type parameter
+ key = None
+ self.skip_ws()
+ if self.skip_word_and_ws('typename'):
+ key = 'typename'
+ elif self.skip_word_and_ws('class'):
+ key = 'class'
+ elif nestedParams:
+ self.fail("Expected 'typename' or 'class' after "
+ "template template parameter list.")
+ else:
+ self.fail("Expected 'typename' or 'class' in the "
+ "beginning of template type parameter.")
+ self.skip_ws()
+ parameterPack = self.skip_string('...')
+ self.skip_ws()
+ if self.match(identifier_re):
+ identifier = ASTIdentifier(self.matched_text)
+ else:
+ identifier = None
+ self.skip_ws()
+ if not parameterPack and self.skip_string('='):
+ default = self._parse_type(named=False, outer=None)
+ else:
+ default = None
+ if self.current_char not in ',>':
+ self.fail('Expected "," or ">" after (template) type parameter.')
+ data = ASTTemplateKeyParamPackIdDefault(key, identifier,
+ parameterPack, default)
+ if nestedParams:
+ return ASTTemplateParamTemplateType(nestedParams, data)
+ else:
+ return ASTTemplateParamType(data)
+ except DefinitionError as eType:
+ if nestedParams:
+ raise
+ try:
+ # non-type parameter or constrained type parameter
+ self.pos = pos
+ param = self._parse_type_with_init('maybe', 'templateParam')
+ self.skip_ws()
+ parameterPack = self.skip_string('...')
+ return ASTTemplateParamNonType(param, parameterPack)
+ except DefinitionError as eNonType:
+ self.pos = pos
+ header = "Error when parsing template parameter."
+ errs = []
+ errs.append(
+ (eType, "If unconstrained type parameter or template type parameter"))
+ errs.append(
+ (eNonType, "If constrained type parameter or non-type parameter"))
+ raise self._make_multi_error(errs, header) from None
+
+ def _parse_template_parameter_list(self) -> ASTTemplateParams:
+ # only: '<' parameter-list '>'
+ # we assume that 'template' has just been parsed
+ templateParams: list[ASTTemplateParam] = []
+ self.skip_ws()
+ if not self.skip_string("<"):
+ self.fail("Expected '<' after 'template'")
+ while 1:
+ pos = self.pos
+ err = None
+ try:
+ param = self._parse_template_parameter()
+ templateParams.append(param)
+ except DefinitionError as eParam:
+ self.pos = pos
+ err = eParam
+ self.skip_ws()
+ if self.skip_string('>'):
+ requiresClause = self._parse_requires_clause()
+ return ASTTemplateParams(templateParams, requiresClause)
+ elif self.skip_string(','):
+ continue
+ else:
+ header = "Error in template parameter list."
+ errs = []
+ if err:
+ errs.append((err, "If parameter"))
+ try:
+ self.fail('Expected "," or ">".')
+ except DefinitionError as e:
+ errs.append((e, "If no parameter"))
+ logger.debug(errs)
+ raise self._make_multi_error(errs, header)
+
+ def _parse_template_introduction(self) -> ASTTemplateIntroduction | None:
+ pos = self.pos
+ try:
+ concept = self._parse_nested_name()
+ except Exception:
+ self.pos = pos
+ return None
+ self.skip_ws()
+ if not self.skip_string('{'):
+ self.pos = pos
+ return None
+
+ # for sure it must be a template introduction now
+ params = []
+ while 1:
+ self.skip_ws()
+ parameterPack = self.skip_string('...')
+ self.skip_ws()
+ if not self.match(identifier_re):
+ self.fail("Expected identifier in template introduction list.")
+ txt_identifier = self.matched_text
+ # make sure there isn't a keyword
+ if txt_identifier in _keywords:
+ self.fail("Expected identifier in template introduction list, "
+ "got keyword: %s" % txt_identifier)
+ identifier = ASTIdentifier(txt_identifier)
+ params.append(ASTTemplateIntroductionParameter(identifier, parameterPack))
+
+ self.skip_ws()
+ if self.skip_string('}'):
+ break
+ if self.skip_string(','):
+ continue
+ self.fail('Error in template introduction list. Expected ",", or "}".')
+ return ASTTemplateIntroduction(concept, params)
+
+ def _parse_requires_clause(self) -> ASTRequiresClause | None:
+ # requires-clause -> 'requires' constraint-logical-or-expression
+ # constraint-logical-or-expression
+ # -> constraint-logical-and-expression
+ # | constraint-logical-or-expression '||' constraint-logical-and-expression
+ # constraint-logical-and-expression
+ # -> primary-expression
+ # | constraint-logical-and-expression '&&' primary-expression
+ self.skip_ws()
+ if not self.skip_word('requires'):
+ return None
+
+ def parse_and_expr(self: DefinitionParser) -> ASTExpression:
+ andExprs = []
+ ops = []
+ andExprs.append(self._parse_primary_expression())
+ while True:
+ self.skip_ws()
+ oneMore = False
+ if self.skip_string('&&'):
+ oneMore = True
+ ops.append('&&')
+ elif self.skip_word('and'):
+ oneMore = True
+ ops.append('and')
+ if not oneMore:
+ break
+ andExprs.append(self._parse_primary_expression())
+ if len(andExprs) == 1:
+ return andExprs[0]
+ else:
+ return ASTBinOpExpr(andExprs, ops)
+
+ orExprs = []
+ ops = []
+ orExprs.append(parse_and_expr(self))
+ while True:
+ self.skip_ws()
+ oneMore = False
+ if self.skip_string('||'):
+ oneMore = True
+ ops.append('||')
+ elif self.skip_word('or'):
+ oneMore = True
+ ops.append('or')
+ if not oneMore:
+ break
+ orExprs.append(parse_and_expr(self))
+ if len(orExprs) == 1:
+ return ASTRequiresClause(orExprs[0])
+ else:
+ return ASTRequiresClause(ASTBinOpExpr(orExprs, ops))
+
+ def _parse_template_declaration_prefix(self, objectType: str,
+ ) -> ASTTemplateDeclarationPrefix | None:
+ templates: list[ASTTemplateParams | ASTTemplateIntroduction] = []
+ while 1:
+ self.skip_ws()
+ # the saved position is only used to provide a better error message
+ params: ASTTemplateParams | ASTTemplateIntroduction | None = None
+ pos = self.pos
+ if self.skip_word("template"):
+ try:
+ params = self._parse_template_parameter_list()
+ except DefinitionError as e:
+ if objectType == 'member' and len(templates) == 0:
+ return ASTTemplateDeclarationPrefix(None)
+ else:
+ raise e
+ if objectType == 'concept' and params.requiresClause is not None:
+ self.fail('requires-clause not allowed for concept')
+ else:
+ params = self._parse_template_introduction()
+ if not params:
+ break
+ if objectType == 'concept' and len(templates) > 0:
+ self.pos = pos
+ self.fail("More than 1 template parameter list for concept.")
+ templates.append(params)
+ if len(templates) == 0 and objectType == 'concept':
+ self.fail('Missing template parameter list for concept.')
+ if len(templates) == 0:
+ return None
+ else:
+ return ASTTemplateDeclarationPrefix(templates)
+
+ def _check_template_consistency(self, nestedName: ASTNestedName,
+ templatePrefix: ASTTemplateDeclarationPrefix,
+ fullSpecShorthand: bool, isMember: bool = False,
+ ) -> ASTTemplateDeclarationPrefix:
+ numArgs = nestedName.num_templates()
+ isMemberInstantiation = False
+ if not templatePrefix:
+ numParams = 0
+ else:
+ if isMember and templatePrefix.templates is None:
+ numParams = 0
+ isMemberInstantiation = True
+ else:
+ numParams = len(templatePrefix.templates)
+ if numArgs + 1 < numParams:
+ self.fail("Too few template argument lists compared to parameter"
+ " lists. Argument lists: %d, Parameter lists: %d."
+ % (numArgs, numParams))
+ if numArgs > numParams:
+ numExtra = numArgs - numParams
+ if not fullSpecShorthand and not isMemberInstantiation:
+ msg = (
+ f'Too many template argument lists compared to parameter lists. '
+ f'Argument lists: {numArgs:d}, Parameter lists: {numParams:d}, '
+ f'Extra empty parameters lists prepended: {numExtra:d}. '
+ 'Declaration:\n\t'
+ )
+ if templatePrefix:
+ msg += f"{templatePrefix}\n\t"
+ msg += str(nestedName)
+ self.warn(msg)
+
+ newTemplates: list[ASTTemplateParams | ASTTemplateIntroduction] = [
+ ASTTemplateParams([], requiresClause=None)
+ for _i in range(numExtra)
+ ]
+ if templatePrefix and not isMemberInstantiation:
+ newTemplates.extend(templatePrefix.templates)
+ templatePrefix = ASTTemplateDeclarationPrefix(newTemplates)
+ return templatePrefix
+
+ def parse_declaration(self, objectType: str, directiveType: str) -> ASTDeclaration:
+ if objectType not in ('class', 'union', 'function', 'member', 'type',
+ 'concept', 'enum', 'enumerator'):
+ raise Exception('Internal error, unknown objectType "%s".' % objectType)
+ if directiveType not in ('class', 'struct', 'union', 'function', 'member', 'var',
+ 'type', 'concept',
+ 'enum', 'enum-struct', 'enum-class', 'enumerator'):
+ raise Exception('Internal error, unknown directiveType "%s".' % directiveType)
+ visibility = None
+ templatePrefix = None
+ trailingRequiresClause = None
+ declaration: Any = None
+
+ self.skip_ws()
+ if self.match(_visibility_re):
+ visibility = self.matched_text
+
+ if objectType in ('type', 'concept', 'member', 'function', 'class', 'union'):
+ templatePrefix = self._parse_template_declaration_prefix(objectType)
+
+ if objectType == 'type':
+ prevErrors = []
+ pos = self.pos
+ try:
+ if not templatePrefix:
+ declaration = self._parse_type(named=True, outer='type')
+ except DefinitionError as e:
+ prevErrors.append((e, "If typedef-like declaration"))
+ self.pos = pos
+ pos = self.pos
+ try:
+ if not declaration:
+ declaration = self._parse_type_using()
+ except DefinitionError as e:
+ self.pos = pos
+ prevErrors.append((e, "If type alias or template alias"))
+ header = "Error in type declaration."
+ raise self._make_multi_error(prevErrors, header) from e
+ elif objectType == 'concept':
+ declaration = self._parse_concept()
+ elif objectType == 'member':
+ declaration = self._parse_type_with_init(named=True, outer='member')
+ elif objectType == 'function':
+ declaration = self._parse_type(named=True, outer='function')
+ trailingRequiresClause = self._parse_requires_clause()
+ elif objectType == 'class':
+ declaration = self._parse_class()
+ elif objectType == 'union':
+ declaration = self._parse_union()
+ elif objectType == 'enum':
+ declaration = self._parse_enum()
+ elif objectType == 'enumerator':
+ declaration = self._parse_enumerator()
+ else:
+ raise AssertionError
+ templatePrefix = self._check_template_consistency(declaration.name,
+ templatePrefix,
+ fullSpecShorthand=False,
+ isMember=objectType == 'member')
+ self.skip_ws()
+ semicolon = self.skip_string(';')
+ return ASTDeclaration(objectType, directiveType, visibility,
+ templatePrefix, declaration,
+ trailingRequiresClause, semicolon)
+
+ def parse_namespace_object(self) -> ASTNamespace:
+ templatePrefix = self._parse_template_declaration_prefix(objectType="namespace")
+ name = self._parse_nested_name()
+ templatePrefix = self._check_template_consistency(name, templatePrefix,
+ fullSpecShorthand=False)
+ res = ASTNamespace(name, templatePrefix)
+ res.objectType = 'namespace' # type: ignore[attr-defined]
+ return res
+
+ def parse_xref_object(self) -> tuple[ASTNamespace | ASTDeclaration, bool]:
+ pos = self.pos
+ try:
+ templatePrefix = self._parse_template_declaration_prefix(objectType="xref")
+ name = self._parse_nested_name()
+ # if there are '()' left, just skip them
+ self.skip_ws()
+ self.skip_string('()')
+ self.assert_end()
+ templatePrefix = self._check_template_consistency(name, templatePrefix,
+ fullSpecShorthand=True)
+ res1 = ASTNamespace(name, templatePrefix)
+ res1.objectType = 'xref' # type: ignore[attr-defined]
+ return res1, True
+ except DefinitionError as e1:
+ try:
+ self.pos = pos
+ res2 = self.parse_declaration('function', 'function')
+ # if there are '()' left, just skip them
+ self.skip_ws()
+ self.skip_string('()')
+ self.assert_end()
+ return res2, False
+ except DefinitionError as e2:
+ errs = []
+ errs.append((e1, "If shorthand ref"))
+ errs.append((e2, "If full function ref"))
+ msg = "Error in cross-reference."
+ raise self._make_multi_error(errs, msg) from e2
+
+ def parse_expression(self) -> ASTExpression | ASTType:
+ pos = self.pos
+ try:
+ expr = self._parse_expression()
+ self.skip_ws()
+ self.assert_end()
+ return expr
+ except DefinitionError as exExpr:
+ self.pos = pos
+ try:
+ typ = self._parse_type(False)
+ self.skip_ws()
+ self.assert_end()
+ return typ
+ except DefinitionError as exType:
+ header = "Error when parsing (type) expression."
+ errs = []
+ errs.append((exExpr, "If expression"))
+ errs.append((exType, "If type"))
+ raise self._make_multi_error(errs, header) from exType
diff --git a/sphinx/domains/cpp/_symbol.py b/sphinx/domains/cpp/_symbol.py
index 1d87ea050..ef5a40588 100644
--- a/sphinx/domains/cpp/_symbol.py
+++ b/sphinx/domains/cpp/_symbol.py
@@ -1,31 +1,44 @@
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any, NoReturn
-from sphinx.domains.cpp._ast import ASTDeclaration, ASTIdentifier, ASTNestedName, ASTNestedNameElement, ASTOperator, ASTTemplateArgs, ASTTemplateDeclarationPrefix, ASTTemplateIntroduction, ASTTemplateParams
+
+from sphinx.domains.cpp._ast import (
+ ASTDeclaration,
+ ASTIdentifier,
+ ASTNestedName,
+ ASTNestedNameElement,
+ ASTOperator,
+ ASTTemplateArgs,
+ ASTTemplateDeclarationPrefix,
+ ASTTemplateIntroduction,
+ ASTTemplateParams,
+)
from sphinx.locale import __
from sphinx.util import logging
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
+
from sphinx.environment import BuildEnvironment
+
logger = logging.getLogger(__name__)
class _DuplicateSymbolError(Exception):
-
- def __init__(self, symbol: Symbol, declaration: ASTDeclaration) ->None:
+ def __init__(self, symbol: Symbol, declaration: ASTDeclaration) -> None:
assert symbol
assert declaration
self.symbol = symbol
self.declaration = declaration
- def __str__(self) ->str:
- return 'Internal C++ duplicate symbol error:\n%s' % self.symbol.dump(0)
+ def __str__(self) -> str:
+ return "Internal C++ duplicate symbol error:\n%s" % self.symbol.dump(0)
class SymbolLookupResult:
-
def __init__(self, symbols: Iterator[Symbol], parentSymbol: Symbol,
- identOrOp: (ASTIdentifier | ASTOperator), templateParams: Any,
- templateArgs: ASTTemplateArgs) ->None:
+ identOrOp: ASTIdentifier | ASTOperator, templateParams: Any,
+ templateArgs: ASTTemplateArgs) -> None:
self.symbols = symbols
self.parentSymbol = parentSymbol
self.identOrOp = identOrOp
@@ -34,57 +47,1049 @@ class SymbolLookupResult:
class LookupKey:
-
- def __init__(self, data: list[tuple[ASTNestedNameElement,
- ASTTemplateParams | ASTTemplateIntroduction, str]]) ->None:
+ def __init__(self, data: list[tuple[ASTNestedNameElement,
+ ASTTemplateParams | ASTTemplateIntroduction,
+ str]]) -> None:
self.data = data
+def _is_specialization(templateParams: ASTTemplateParams | ASTTemplateIntroduction,
+ templateArgs: ASTTemplateArgs) -> bool:
+ # Checks if `templateArgs` does not exactly match `templateParams`.
+ # the names of the template parameters must be given exactly as args
+ # and params that are packs must in the args be the name expanded
+ if len(templateParams.params) != len(templateArgs.args):
+ return True
+ # having no template params and no arguments is also a specialization
+ if len(templateParams.params) == 0:
+ return True
+ for i in range(len(templateParams.params)):
+ param = templateParams.params[i]
+ arg = templateArgs.args[i]
+ # TODO: doing this by string manipulation is probably not the most efficient
+ paramName = str(param.name)
+ argTxt = str(arg)
+ isArgPackExpansion = argTxt.endswith('...')
+ if param.isPack != isArgPackExpansion:
+ return True
+ argName = argTxt[:-3] if isArgPackExpansion else argTxt
+ if paramName != argName:
+ return True
+ return False
+
+
class Symbol:
debug_indent = 0
- debug_indent_string = ' '
- debug_lookup = False
- debug_show_tree = False
+ debug_indent_string = " "
+ debug_lookup = False # overridden by the corresponding config value
+ debug_show_tree = False # overridden by the corresponding config value
- def __copy__(self) ->NoReturn:
- raise AssertionError
+ def __copy__(self) -> NoReturn:
+ raise AssertionError # shouldn't happen
- def __deepcopy__(self, memo: Any) ->Symbol:
+ def __deepcopy__(self, memo: Any) -> Symbol:
if self.parent:
- raise AssertionError
+ raise AssertionError # shouldn't happen
+ # the domain base class makes a copy of the initial data, which is fine
return Symbol(None, None, None, None, None, None, None)
- def __setattr__(self, key: str, value: Any) ->None:
- if key == 'children':
+ @staticmethod
+ def debug_print(*args: Any) -> None:
+ logger.debug(Symbol.debug_indent_string * Symbol.debug_indent, end="")
+ logger.debug(*args)
+
+ def _assert_invariants(self) -> None:
+ if not self.parent:
+ # parent == None means global scope, so declaration means a parent
+ assert not self.identOrOp
+ assert not self.templateParams
+ assert not self.templateArgs
+ assert not self.declaration
+ assert not self.docname
+ else:
+ if self.declaration:
+ assert self.docname
+
+ def __setattr__(self, key: str, value: Any) -> None:
+ if key == "children":
raise AssertionError
return super().__setattr__(key, value)
- def __init__(self, parent: (Symbol | None), identOrOp: (ASTIdentifier |
- ASTOperator | None), templateParams: (ASTTemplateParams |
- ASTTemplateIntroduction | None), templateArgs: Any, declaration: (
- ASTDeclaration | None), docname: (str | None), line: (int | None)
- ) ->None:
+ def __init__(self, parent: Symbol | None,
+ identOrOp: ASTIdentifier | ASTOperator | None,
+ templateParams: ASTTemplateParams | ASTTemplateIntroduction | None,
+ templateArgs: Any, declaration: ASTDeclaration | None,
+ docname: str | None, line: int | None) -> None:
self.parent = parent
+ # declarations in a single directive are linked together
self.siblingAbove: Symbol | None = None
self.siblingBelow: Symbol | None = None
self.identOrOp = identOrOp
- if templateArgs is not None and not _is_specialization(templateParams,
- templateArgs):
+ # Ensure the same symbol for `A` is created for:
+ #
+ # .. cpp:class:: template <typename T> class A
+ #
+ # and
+ #
+ # .. cpp:function:: template <typename T> int A<T>::foo()
+ if (templateArgs is not None and
+ not _is_specialization(templateParams, templateArgs)):
templateArgs = None
- self.templateParams = templateParams
- self.templateArgs = templateArgs
+ self.templateParams = templateParams # template<templateParams>
+ self.templateArgs = templateArgs # identifier<templateArgs>
self.declaration = declaration
self.docname = docname
self.line = line
self.isRedeclaration = False
self._assert_invariants()
+
+ # Remember to modify Symbol.remove if modifications to the parent change.
self._children: list[Symbol] = []
self._anonChildren: list[Symbol] = []
+ # note: _children includes _anonChildren
if self.parent:
self.parent._children.append(self)
if self.declaration:
self.declaration.symbol = self
+
+ # Do symbol addition after self._children has been initialised.
self._add_template_and_function_params()
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return f'<Symbol {self.to_string(indent=0)!r}>'
+
+ def _fill_empty(self, declaration: ASTDeclaration, docname: str, line: int) -> None:
+ self._assert_invariants()
+ assert self.declaration is None
+ assert self.docname is None
+ assert self.line is None
+ assert declaration is not None
+ assert docname is not None
+ assert line is not None
+ self.declaration = declaration
+ self.declaration.symbol = self
+ self.docname = docname
+ self.line = line
+ self._assert_invariants()
+ # and symbol addition should be done as well
+ self._add_template_and_function_params()
+
+ def _add_template_and_function_params(self) -> None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_add_template_and_function_params:")
+ # Note: we may be called from _fill_empty, so the symbols we want
+ # to add may actually already be present (as empty symbols).
+
+ # add symbols for the template params
+ if self.templateParams:
+ for tp in self.templateParams.params:
+ if not tp.get_identifier():
+ continue
+ # only add a declaration if we our self are from a declaration
+ if self.declaration:
+ decl = ASTDeclaration(objectType='templateParam', declaration=tp)
+ else:
+ decl = None
+ nne = ASTNestedNameElement(tp.get_identifier(), None)
+ nn = ASTNestedName([nne], [False], rooted=False)
+ self._add_symbols(nn, [], decl, self.docname, self.line)
+ # add symbols for function parameters, if any
+ if self.declaration is not None and self.declaration.function_params is not None:
+ for fp in self.declaration.function_params:
+ if fp.arg is None:
+ continue
+ nn = fp.arg.name
+ if nn is None:
+ continue
+ # (comparing to the template params: we have checked that we are a declaration)
+ decl = ASTDeclaration(objectType='functionParam', declaration=fp)
+ assert not nn.rooted
+ assert len(nn.names) == 1
+ self._add_symbols(nn, [], decl, self.docname, self.line)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+
+ def remove(self) -> None:
+ if self.parent is None:
+ return
+ assert self in self.parent._children
+ self.parent._children.remove(self)
+ self.parent = None
+
+ def clear_doc(self, docname: str) -> None:
+ newChildren: list[Symbol] = []
+ for sChild in self._children:
+ sChild.clear_doc(docname)
+ if sChild.declaration and sChild.docname == docname:
+ sChild.declaration = None
+ sChild.docname = None
+ sChild.line = None
+ if sChild.siblingAbove is not None:
+ sChild.siblingAbove.siblingBelow = sChild.siblingBelow
+ if sChild.siblingBelow is not None:
+ sChild.siblingBelow.siblingAbove = sChild.siblingAbove
+ sChild.siblingAbove = None
+ sChild.siblingBelow = None
+ newChildren.append(sChild)
+ self._children = newChildren
+
+ def get_all_symbols(self) -> Iterator[Any]:
+ yield self
+ for sChild in self._children:
+ yield from sChild.get_all_symbols()
+
+ @property
+ def children_recurse_anon(self) -> Iterator[Symbol]:
+ for c in self._children:
+ yield c
+ if not c.identOrOp.is_anon():
+ continue
+
+ yield from c.children_recurse_anon
+
+ def get_lookup_key(self) -> LookupKey:
+ # The pickle files for the environment and for each document are distinct.
+ # The environment has all the symbols, but the documents has xrefs that
+ # must know their scope. A lookup key is essentially a specification of
+ # how to find a specific symbol.
+ symbols = []
+ s = self
+ while s.parent:
+ symbols.append(s)
+ s = s.parent
+ symbols.reverse()
+ key = []
+ for s in symbols:
+ nne = ASTNestedNameElement(s.identOrOp, s.templateArgs)
+ if s.declaration is not None:
+ key.append((nne, s.templateParams, s.declaration.get_newest_id()))
+ else:
+ key.append((nne, s.templateParams, None))
+ return LookupKey(key)
+
+ def get_full_nested_name(self) -> ASTNestedName:
+ symbols = []
+ s = self
+ while s.parent:
+ symbols.append(s)
+ s = s.parent
+ symbols.reverse()
+ names = []
+ templates = []
+ for s in symbols:
+ names.append(ASTNestedNameElement(s.identOrOp, s.templateArgs))
+ templates.append(False)
+ return ASTNestedName(names, templates, rooted=False)
+
+ def _find_first_named_symbol(self, identOrOp: ASTIdentifier | ASTOperator,
+ templateParams: ASTTemplateParams | ASTTemplateIntroduction,
+ templateArgs: ASTTemplateArgs | None,
+ templateShorthand: bool, matchSelf: bool,
+ recurseInAnon: bool, correctPrimaryTemplateArgs: bool,
+ ) -> Symbol | None:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("_find_first_named_symbol ->")
+ res = self._find_named_symbols(identOrOp, templateParams, templateArgs,
+ templateShorthand, matchSelf, recurseInAnon,
+ correctPrimaryTemplateArgs,
+ searchInSiblings=False)
+ try:
+ return next(res)
+ except StopIteration:
+ return None
+
+ def _find_named_symbols(self, identOrOp: ASTIdentifier | ASTOperator,
+ templateParams: ASTTemplateParams | ASTTemplateIntroduction,
+ templateArgs: ASTTemplateArgs,
+ templateShorthand: bool, matchSelf: bool,
+ recurseInAnon: bool, correctPrimaryTemplateArgs: bool,
+ searchInSiblings: bool) -> Iterator[Symbol]:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_find_named_symbols:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("self:")
+ logger.debug(self.to_string(Symbol.debug_indent + 1), end="")
+ Symbol.debug_print("identOrOp: ", identOrOp)
+ Symbol.debug_print("templateParams: ", templateParams)
+ Symbol.debug_print("templateArgs: ", templateArgs)
+ Symbol.debug_print("templateShorthand: ", templateShorthand)
+ Symbol.debug_print("matchSelf: ", matchSelf)
+ Symbol.debug_print("recurseInAnon: ", recurseInAnon)
+ Symbol.debug_print("correctPrimaryTemplateAargs:", correctPrimaryTemplateArgs)
+ Symbol.debug_print("searchInSiblings: ", searchInSiblings)
+
+ if correctPrimaryTemplateArgs:
+ if templateParams is not None and templateArgs is not None:
+ # If both are given, but it's not a specialization, then do lookup as if
+ # there is no argument list.
+ # For example: template<typename T> int A<T>::var;
+ if not _is_specialization(templateParams, templateArgs):
+ templateArgs = None
+
+ def matches(s: Symbol) -> bool:
+ if s.identOrOp != identOrOp:
+ return False
+ if (s.templateParams is None) != (templateParams is None):
+ if templateParams is not None:
+ # we query with params, they must match params
+ return False
+ if not templateShorthand:
+ # we don't query with params, and we do care about them
+ return False
+ if templateParams:
+ # TODO: do better comparison
+ if str(s.templateParams) != str(templateParams):
+ return False
+ if (s.templateArgs is None) != (templateArgs is None):
+ return False
+ if s.templateArgs:
+ # TODO: do better comparison
+ if str(s.templateArgs) != str(templateArgs):
+ return False
+ return True
+
+ def candidates() -> Iterator[Symbol]:
+ s = self
+ if Symbol.debug_lookup:
+ Symbol.debug_print("searching in self:")
+ logger.debug(s.to_string(Symbol.debug_indent + 1), end="")
+ while True:
+ if matchSelf:
+ yield s
+ if recurseInAnon:
+ yield from s.children_recurse_anon
+ else:
+ yield from s._children
+
+ if s.siblingAbove is None:
+ break
+ s = s.siblingAbove
+ if Symbol.debug_lookup:
+ Symbol.debug_print("searching in sibling:")
+ logger.debug(s.to_string(Symbol.debug_indent + 1), end="")
+
+ for s in candidates():
+ if Symbol.debug_lookup:
+ Symbol.debug_print("candidate:")
+ logger.debug(s.to_string(Symbol.debug_indent + 1), end="")
+ if matches(s):
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("matches")
+ Symbol.debug_indent -= 3
+ yield s
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 2
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+
+ def _symbol_lookup(
+ self,
+ nestedName: ASTNestedName,
+ templateDecls: list[Any],
+ onMissingQualifiedSymbol: Callable[
+ [Symbol, ASTIdentifier | ASTOperator, Any, ASTTemplateArgs], Symbol | None,
+ ],
+ strictTemplateParamArgLists: bool, ancestorLookupType: str,
+ templateShorthand: bool, matchSelf: bool,
+ recurseInAnon: bool, correctPrimaryTemplateArgs: bool,
+ searchInSiblings: bool,
+ ) -> SymbolLookupResult:
+ # ancestorLookupType: if not None, specifies the target type of the lookup
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_symbol_lookup:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("self:")
+ logger.debug(self.to_string(Symbol.debug_indent + 1), end="")
+ Symbol.debug_print("nestedName: ", nestedName)
+ Symbol.debug_print("templateDecls: ", ",".join(str(t) for t in templateDecls))
+ Symbol.debug_print("strictTemplateParamArgLists:", strictTemplateParamArgLists)
+ Symbol.debug_print("ancestorLookupType:", ancestorLookupType)
+ Symbol.debug_print("templateShorthand: ", templateShorthand)
+ Symbol.debug_print("matchSelf: ", matchSelf)
+ Symbol.debug_print("recurseInAnon: ", recurseInAnon)
+ Symbol.debug_print("correctPrimaryTemplateArgs: ", correctPrimaryTemplateArgs)
+ Symbol.debug_print("searchInSiblings: ", searchInSiblings)
+
+ if strictTemplateParamArgLists:
+ # Each template argument list must have a template parameter list.
+ # But to declare a template there must be an additional template parameter list.
+ assert (nestedName.num_templates() == len(templateDecls) or
+ nestedName.num_templates() + 1 == len(templateDecls))
+ else:
+ assert len(templateDecls) <= nestedName.num_templates() + 1
+
+ names = nestedName.names
+
+ # find the right starting point for lookup
+ parentSymbol = self
+ if nestedName.rooted:
+ while parentSymbol.parent:
+ parentSymbol = parentSymbol.parent
+ if ancestorLookupType is not None:
+ # walk up until we find the first identifier
+ firstName = names[0]
+ if not firstName.is_operator():
+ while parentSymbol.parent:
+ if parentSymbol.find_identifier(firstName.identOrOp,
+ matchSelf=matchSelf,
+ recurseInAnon=recurseInAnon,
+ searchInSiblings=searchInSiblings):
+ # if we are in the scope of a constructor but wants to
+ # reference the class we need to walk one extra up
+ if (len(names) == 1 and ancestorLookupType == 'class' and matchSelf and
+ parentSymbol.parent and
+ parentSymbol.parent.identOrOp == firstName.identOrOp):
+ pass
+ else:
+ break
+ parentSymbol = parentSymbol.parent
+
+ if Symbol.debug_lookup:
+ Symbol.debug_print("starting point:")
+ logger.debug(parentSymbol.to_string(Symbol.debug_indent + 1), end="")
+
+ # and now the actual lookup
+ iTemplateDecl = 0
+ for name in names[:-1]:
+ identOrOp = name.identOrOp
+ templateArgs = name.templateArgs
+ if strictTemplateParamArgLists:
+ # there must be a parameter list
+ if templateArgs:
+ assert iTemplateDecl < len(templateDecls)
+ templateParams = templateDecls[iTemplateDecl]
+ iTemplateDecl += 1
+ else:
+ templateParams = None
+ else:
+ # take the next template parameter list if there is one
+ # otherwise it's ok
+ if templateArgs and iTemplateDecl < len(templateDecls):
+ templateParams = templateDecls[iTemplateDecl]
+ iTemplateDecl += 1
+ else:
+ templateParams = None
+
+ symbol = parentSymbol._find_first_named_symbol(
+ identOrOp,
+ templateParams, templateArgs,
+ templateShorthand=templateShorthand,
+ matchSelf=matchSelf,
+ recurseInAnon=recurseInAnon,
+ correctPrimaryTemplateArgs=correctPrimaryTemplateArgs)
+ if symbol is None:
+ symbol = onMissingQualifiedSymbol(parentSymbol, identOrOp,
+ templateParams, templateArgs)
+ if symbol is None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return None
+ # We have now matched part of a nested name, and need to match more
+ # so even if we should matchSelf before, we definitely shouldn't
+ # even more. (see also issue #2666)
+ matchSelf = False
+ parentSymbol = symbol
+
+ if Symbol.debug_lookup:
+ Symbol.debug_print("handle last name from:")
+ logger.debug(parentSymbol.to_string(Symbol.debug_indent + 1), end="")
+
+ # handle the last name
+ name = names[-1]
+ identOrOp = name.identOrOp
+ templateArgs = name.templateArgs
+ if iTemplateDecl < len(templateDecls):
+ assert iTemplateDecl + 1 == len(templateDecls)
+ templateParams = templateDecls[iTemplateDecl]
+ else:
+ assert iTemplateDecl == len(templateDecls)
+ templateParams = None
+
+ symbols = parentSymbol._find_named_symbols(
+ identOrOp, templateParams, templateArgs,
+ templateShorthand=templateShorthand, matchSelf=matchSelf,
+ recurseInAnon=recurseInAnon, correctPrimaryTemplateArgs=False,
+ searchInSiblings=searchInSiblings)
+ if Symbol.debug_lookup:
+ symbols = list(symbols) # type: ignore[assignment]
+ Symbol.debug_indent -= 2
+ return SymbolLookupResult(symbols, parentSymbol,
+ identOrOp, templateParams, templateArgs)
+
+ def _add_symbols(
+ self,
+ nestedName: ASTNestedName,
+ templateDecls: list[Any],
+ declaration: ASTDeclaration | None,
+ docname: str | None,
+ line: int | None,
+ ) -> Symbol:
+ # Used for adding a whole path of symbols, where the last may or may not
+ # be an actual declaration.
+
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_add_symbols:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("tdecls:", ",".join(str(t) for t in templateDecls))
+ Symbol.debug_print("nn: ", nestedName)
+ Symbol.debug_print("decl: ", declaration)
+ Symbol.debug_print(f"location: {docname}:{line}")
+
+ def onMissingQualifiedSymbol(parentSymbol: Symbol,
+ identOrOp: ASTIdentifier | ASTOperator,
+ templateParams: Any, templateArgs: ASTTemplateArgs,
+ ) -> Symbol | None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("_add_symbols, onMissingQualifiedSymbol:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("templateParams:", templateParams)
+ Symbol.debug_print("identOrOp: ", identOrOp)
+ Symbol.debug_print("templateARgs: ", templateArgs)
+ Symbol.debug_indent -= 2
+ return Symbol(parent=parentSymbol, identOrOp=identOrOp,
+ templateParams=templateParams,
+ templateArgs=templateArgs, declaration=None,
+ docname=None, line=None)
+
+ lookupResult = self._symbol_lookup(nestedName, templateDecls,
+ onMissingQualifiedSymbol,
+ strictTemplateParamArgLists=True,
+ ancestorLookupType=None,
+ templateShorthand=False,
+ matchSelf=False,
+ recurseInAnon=False,
+ correctPrimaryTemplateArgs=True,
+ searchInSiblings=False)
+ assert lookupResult is not None # we create symbols all the way, so that can't happen
+ symbols = list(lookupResult.symbols)
+ if len(symbols) == 0:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("_add_symbols, result, no symbol:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("templateParams:", lookupResult.templateParams)
+ Symbol.debug_print("identOrOp: ", lookupResult.identOrOp)
+ Symbol.debug_print("templateArgs: ", lookupResult.templateArgs)
+ Symbol.debug_print("declaration: ", declaration)
+ Symbol.debug_print(f"location: {docname}:{line}")
+ Symbol.debug_indent -= 1
+ symbol = Symbol(parent=lookupResult.parentSymbol,
+ identOrOp=lookupResult.identOrOp,
+ templateParams=lookupResult.templateParams,
+ templateArgs=lookupResult.templateArgs,
+ declaration=declaration,
+ docname=docname, line=line)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return symbol
+
+ if Symbol.debug_lookup:
+ Symbol.debug_print("_add_symbols, result, symbols:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("number symbols:", len(symbols))
+ Symbol.debug_indent -= 1
+
+ if not declaration:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("no declaration")
+ Symbol.debug_indent -= 2
+ # good, just a scope creation
+ # TODO: what if we have more than one symbol?
+ return symbols[0]
+
+ noDecl = []
+ withDecl = []
+ dupDecl = []
+ for s in symbols:
+ if s.declaration is None:
+ noDecl.append(s)
+ elif s.isRedeclaration:
+ dupDecl.append(s)
+ else:
+ withDecl.append(s)
+ if Symbol.debug_lookup:
+ Symbol.debug_print("#noDecl: ", len(noDecl))
+ Symbol.debug_print("#withDecl:", len(withDecl))
+ Symbol.debug_print("#dupDecl: ", len(dupDecl))
+ # With partial builds we may start with a large symbol tree stripped of declarations.
+ # Essentially any combination of noDecl, withDecl, and dupDecls seems possible.
+ # TODO: make partial builds fully work. What should happen when the primary symbol gets
+ # deleted, and other duplicates exist? The full document should probably be rebuild.
+
+ # First check if one of those with a declaration matches.
+ # If it's a function, we need to compare IDs,
+ # otherwise there should be only one symbol with a declaration.
+ def makeCandSymbol() -> Symbol:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("begin: creating candidate symbol")
+ symbol = Symbol(parent=lookupResult.parentSymbol,
+ identOrOp=lookupResult.identOrOp,
+ templateParams=lookupResult.templateParams,
+ templateArgs=lookupResult.templateArgs,
+ declaration=declaration,
+ docname=docname, line=line)
+ if Symbol.debug_lookup:
+ Symbol.debug_print("end: creating candidate symbol")
+ return symbol
+ if len(withDecl) == 0:
+ candSymbol = None
+ else:
+ candSymbol = makeCandSymbol()
+
+ def handleDuplicateDeclaration(symbol: Symbol, candSymbol: Symbol) -> None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("redeclaration")
+ Symbol.debug_indent -= 1
+ Symbol.debug_indent -= 2
+ # Redeclaration of the same symbol.
+ # Let the new one be there, but raise an error to the client
+ # so it can use the real symbol as subscope.
+ # This will probably result in a duplicate id warning.
+ candSymbol.isRedeclaration = True
+ raise _DuplicateSymbolError(symbol, declaration)
+
+ if declaration.objectType != "function":
+ assert len(withDecl) <= 1
+ handleDuplicateDeclaration(withDecl[0], candSymbol)
+ # (not reachable)
+
+ # a function, so compare IDs
+ candId = declaration.get_newest_id()
+ if Symbol.debug_lookup:
+ Symbol.debug_print("candId:", candId)
+ for symbol in withDecl:
+ # but all existing must be functions as well,
+ # otherwise we declare it to be a duplicate
+ if symbol.declaration.objectType != 'function':
+ handleDuplicateDeclaration(symbol, candSymbol)
+ # (not reachable)
+ oldId = symbol.declaration.get_newest_id()
+ if Symbol.debug_lookup:
+ Symbol.debug_print("oldId: ", oldId)
+ if candId == oldId:
+ handleDuplicateDeclaration(symbol, candSymbol)
+ # (not reachable)
+ # no candidate symbol found with matching ID
+ # if there is an empty symbol, fill that one
+ if len(noDecl) == 0:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("no match, no empty")
+ if candSymbol is not None:
+ Symbol.debug_print("result is already created candSymbol")
+ else:
+ Symbol.debug_print("result is makeCandSymbol()")
+ Symbol.debug_indent -= 2
+ if candSymbol is not None:
+ return candSymbol
+ else:
+ return makeCandSymbol()
+ else:
+ if Symbol.debug_lookup:
+ Symbol.debug_print(
+ "no match, but fill an empty declaration, candSybmol is not None?:",
+ candSymbol is not None,
+ )
+ Symbol.debug_indent -= 2
+ if candSymbol is not None:
+ candSymbol.remove()
+ # assert len(noDecl) == 1
+ # TODO: enable assertion when we at some point find out how to do cleanup
+ # for now, just take the first one, it should work fine ... right?
+ symbol = noDecl[0]
+ # If someone first opened the scope, and then later
+ # declares it, e.g,
+ # .. namespace:: Test
+ # .. namespace:: nullptr
+ # .. class:: Test
+ symbol._fill_empty(declaration, docname, line)
+ return symbol
+
+ def merge_with(self, other: Symbol, docnames: list[str],
+ env: BuildEnvironment) -> None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("merge_with:")
+ assert other is not None
+
+ def unconditionalAdd(self: Symbol, otherChild: Symbol) -> None:
+ # TODO: hmm, should we prune by docnames?
+ self._children.append(otherChild)
+ otherChild.parent = self
+ otherChild._assert_invariants()
+
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ for otherChild in other._children:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("otherChild:\n", otherChild.to_string(Symbol.debug_indent))
+ Symbol.debug_indent += 1
+ if otherChild.isRedeclaration:
+ unconditionalAdd(self, otherChild)
+ if Symbol.debug_lookup:
+ Symbol.debug_print("isRedeclaration")
+ Symbol.debug_indent -= 1
+ continue
+ candiateIter = self._find_named_symbols(
+ identOrOp=otherChild.identOrOp,
+ templateParams=otherChild.templateParams,
+ templateArgs=otherChild.templateArgs,
+ templateShorthand=False, matchSelf=False,
+ recurseInAnon=False, correctPrimaryTemplateArgs=False,
+ searchInSiblings=False)
+ candidates = list(candiateIter)
+
+ if Symbol.debug_lookup:
+ Symbol.debug_print("raw candidate symbols:", len(candidates))
+ symbols = [s for s in candidates if not s.isRedeclaration]
+ if Symbol.debug_lookup:
+ Symbol.debug_print("non-duplicate candidate symbols:", len(symbols))
+
+ if len(symbols) == 0:
+ unconditionalAdd(self, otherChild)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+ continue
+
+ ourChild = None
+ if otherChild.declaration is None:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("no declaration in other child")
+ ourChild = symbols[0]
+ else:
+ queryId = otherChild.declaration.get_newest_id()
+ if Symbol.debug_lookup:
+ Symbol.debug_print("queryId: ", queryId)
+ for symbol in symbols:
+ if symbol.declaration is None:
+ if Symbol.debug_lookup:
+ Symbol.debug_print("empty candidate")
+ # if in the end we have non-matching, but have an empty one,
+ # then just continue with that
+ ourChild = symbol
+ continue
+ candId = symbol.declaration.get_newest_id()
+ if Symbol.debug_lookup:
+ Symbol.debug_print("candidate:", candId)
+ if candId == queryId:
+ ourChild = symbol
+ break
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+ if ourChild is None:
+ unconditionalAdd(self, otherChild)
+ continue
+ if otherChild.declaration and otherChild.docname in docnames:
+ if not ourChild.declaration:
+ ourChild._fill_empty(otherChild.declaration,
+ otherChild.docname, otherChild.line)
+ elif ourChild.docname != otherChild.docname:
+ name = str(ourChild.declaration)
+ msg = __("Duplicate C++ declaration, also defined at %s:%s.\n"
+ "Declaration is '.. cpp:%s:: %s'.")
+ msg = msg % (ourChild.docname, ourChild.line,
+ ourChild.declaration.directiveType, name)
+ logger.warning(msg, location=(otherChild.docname, otherChild.line))
+ else:
+ if (otherChild.declaration.objectType ==
+ ourChild.declaration.objectType and
+ otherChild.declaration.objectType in
+ ('templateParam', 'functionParam') and
+ ourChild.parent.declaration == otherChild.parent.declaration):
+ # `ourChild` was just created during merging by the call
+ # to `_fill_empty` on the parent and can be ignored.
+ pass
+ else:
+ # Both have declarations, and in the same docname.
+ # This can apparently happen, it should be safe to
+ # just ignore it, right?
+ # Hmm, only on duplicate declarations, right?
+ msg = "Internal C++ domain error during symbol merging.\n"
+ msg += "ourChild:\n" + ourChild.to_string(1)
+ msg += "\notherChild:\n" + otherChild.to_string(1)
+ logger.warning(msg, location=otherChild.docname)
+ ourChild.merge_with(otherChild, docnames, env)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+
+ def add_name(self, nestedName: ASTNestedName,
+ templatePrefix: ASTTemplateDeclarationPrefix | None = None) -> Symbol:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("add_name:")
+ if templatePrefix:
+ templateDecls = templatePrefix.templates
+ else:
+ templateDecls = []
+ res = self._add_symbols(nestedName, templateDecls,
+ declaration=None, docname=None, line=None)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+ return res
+
+ def add_declaration(self, declaration: ASTDeclaration,
+ docname: str, line: int) -> Symbol:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("add_declaration:")
+ assert declaration is not None
+ assert docname is not None
+ assert line is not None
+ nestedName = declaration.name
+ if declaration.templatePrefix:
+ templateDecls = declaration.templatePrefix.templates
+ else:
+ templateDecls = []
+ res = self._add_symbols(nestedName, templateDecls, declaration, docname, line)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+ return res
+
+ def find_identifier(self, identOrOp: ASTIdentifier | ASTOperator,
+ matchSelf: bool, recurseInAnon: bool, searchInSiblings: bool,
+ ) -> Symbol | None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("find_identifier:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("identOrOp: ", identOrOp)
+ Symbol.debug_print("matchSelf: ", matchSelf)
+ Symbol.debug_print("recurseInAnon: ", recurseInAnon)
+ Symbol.debug_print("searchInSiblings:", searchInSiblings)
+ logger.debug(self.to_string(Symbol.debug_indent + 1), end="")
+ Symbol.debug_indent -= 2
+ current = self
+ while current is not None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 2
+ Symbol.debug_print("trying:")
+ logger.debug(current.to_string(Symbol.debug_indent + 1), end="")
+ Symbol.debug_indent -= 2
+ if matchSelf and current.identOrOp == identOrOp:
+ return current
+ children = current.children_recurse_anon if recurseInAnon else current._children
+ for s in children:
+ if s.identOrOp == identOrOp:
+ return s
+ if not searchInSiblings:
+ break
+ current = current.siblingAbove
+ return None
+
+ def direct_lookup(self, key: LookupKey) -> Symbol:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("direct_lookup:")
+ Symbol.debug_indent += 1
+ s = self
+ for name, templateParams, id_ in key.data:
+ if id_ is not None:
+ res = None
+ for cand in s._children:
+ if cand.declaration is None:
+ continue
+ if cand.declaration.get_newest_id() == id_:
+ res = cand
+ break
+ s = res
+ else:
+ identOrOp = name.identOrOp
+ templateArgs = name.templateArgs
+ s = s._find_first_named_symbol(identOrOp,
+ templateParams, templateArgs,
+ templateShorthand=False,
+ matchSelf=False,
+ recurseInAnon=False,
+ correctPrimaryTemplateArgs=False)
+ if Symbol.debug_lookup:
+ Symbol.debug_print("name: ", name)
+ Symbol.debug_print("templateParams:", templateParams)
+ Symbol.debug_print("id: ", id_)
+ if s is not None:
+ logger.debug(s.to_string(Symbol.debug_indent + 1), end="")
+ else:
+ Symbol.debug_print("not found")
+ if s is None:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return None
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return s
+
+ def find_name(
+ self,
+ nestedName: ASTNestedName,
+ templateDecls: list[Any],
+ typ: str,
+ templateShorthand: bool,
+ matchSelf: bool,
+ recurseInAnon: bool,
+ searchInSiblings: bool,
+ ) -> tuple[list[Symbol] | None, str]:
+ # templateShorthand: missing template parameter lists for templates is ok
+ # If the first component is None,
+ # then the second component _may_ be a string explaining why.
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("find_name:")
+ Symbol.debug_indent += 1
+ Symbol.debug_print("self:")
+ logger.debug(self.to_string(Symbol.debug_indent + 1), end="")
+ Symbol.debug_print("nestedName: ", nestedName)
+ Symbol.debug_print("templateDecls: ", templateDecls)
+ Symbol.debug_print("typ: ", typ)
+ Symbol.debug_print("templateShorthand:", templateShorthand)
+ Symbol.debug_print("matchSelf: ", matchSelf)
+ Symbol.debug_print("recurseInAnon: ", recurseInAnon)
+ Symbol.debug_print("searchInSiblings: ", searchInSiblings)
+
+ class QualifiedSymbolIsTemplateParam(Exception):
+ pass
+
+ def onMissingQualifiedSymbol(parentSymbol: Symbol,
+ identOrOp: ASTIdentifier | ASTOperator,
+ templateParams: Any,
+ templateArgs: ASTTemplateArgs) -> Symbol | None:
+ # TODO: Maybe search without template args?
+ # Though, the correctPrimaryTemplateArgs does
+ # that for primary templates.
+ # Is there another case where it would be good?
+ if parentSymbol.declaration is not None:
+ if parentSymbol.declaration.objectType == 'templateParam':
+ raise QualifiedSymbolIsTemplateParam
+ return None
+
+ try:
+ lookupResult = self._symbol_lookup(nestedName, templateDecls,
+ onMissingQualifiedSymbol,
+ strictTemplateParamArgLists=False,
+ ancestorLookupType=typ,
+ templateShorthand=templateShorthand,
+ matchSelf=matchSelf,
+ recurseInAnon=recurseInAnon,
+ correctPrimaryTemplateArgs=False,
+ searchInSiblings=searchInSiblings)
+ except QualifiedSymbolIsTemplateParam:
+ return None, "templateParamInQualified"
+
+ if lookupResult is None:
+ # if it was a part of the qualification that could not be found
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return None, None
+
+ res = list(lookupResult.symbols)
+ if len(res) != 0:
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ return res, None
+
+ if lookupResult.parentSymbol.declaration is not None:
+ if lookupResult.parentSymbol.declaration.objectType == 'templateParam':
+ return None, "templateParamInQualified"
+
+ # try without template params and args
+ symbol = lookupResult.parentSymbol._find_first_named_symbol(
+ lookupResult.identOrOp, None, None,
+ templateShorthand=templateShorthand, matchSelf=matchSelf,
+ recurseInAnon=recurseInAnon, correctPrimaryTemplateArgs=False)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 2
+ if symbol is not None:
+ return [symbol], None
+ else:
+ return None, None
+
+ def find_declaration(self, declaration: ASTDeclaration, typ: str, templateShorthand: bool,
+ matchSelf: bool, recurseInAnon: bool) -> Symbol | None:
+ # templateShorthand: missing template parameter lists for templates is ok
+ if Symbol.debug_lookup:
+ Symbol.debug_indent += 1
+ Symbol.debug_print("find_declaration:")
+ nestedName = declaration.name
+ if declaration.templatePrefix:
+ templateDecls = declaration.templatePrefix.templates
+ else:
+ templateDecls = []
+
+ def onMissingQualifiedSymbol(parentSymbol: Symbol,
+ identOrOp: ASTIdentifier | ASTOperator,
+ templateParams: Any,
+ templateArgs: ASTTemplateArgs) -> Symbol | None:
+ return None
+
+ lookupResult = self._symbol_lookup(nestedName, templateDecls,
+ onMissingQualifiedSymbol,
+ strictTemplateParamArgLists=False,
+ ancestorLookupType=typ,
+ templateShorthand=templateShorthand,
+ matchSelf=matchSelf,
+ recurseInAnon=recurseInAnon,
+ correctPrimaryTemplateArgs=False,
+ searchInSiblings=False)
+ if Symbol.debug_lookup:
+ Symbol.debug_indent -= 1
+ if lookupResult is None:
+ return None
+
+ symbols = list(lookupResult.symbols)
+ if len(symbols) == 0:
+ return None
+
+ querySymbol = Symbol(parent=lookupResult.parentSymbol,
+ identOrOp=lookupResult.identOrOp,
+ templateParams=lookupResult.templateParams,
+ templateArgs=lookupResult.templateArgs,
+ declaration=declaration,
+ docname='fakeDocnameForQuery',
+ line=42)
+ queryId = declaration.get_newest_id()
+ for symbol in symbols:
+ if symbol.declaration is None:
+ continue
+ candId = symbol.declaration.get_newest_id()
+ if candId == queryId:
+ querySymbol.remove()
+ return symbol
+ querySymbol.remove()
+ return None
+
+ def to_string(self, indent: int) -> str:
+ res = [Symbol.debug_indent_string * indent]
+ if not self.parent:
+ res.append('::')
+ else:
+ if self.templateParams:
+ res.append(str(self.templateParams))
+ res.append('\n')
+ res.append(Symbol.debug_indent_string * indent)
+ if self.identOrOp:
+ res.append(str(self.identOrOp))
+ else:
+ res.append(str(self.declaration))
+ if self.templateArgs:
+ res.append(str(self.templateArgs))
+ if self.declaration:
+ res.append(": ")
+ if self.isRedeclaration:
+ res.append('!!duplicate!! ')
+ res.append("{" + self.declaration.objectType + "} ")
+ res.append(str(self.declaration))
+ if self.docname:
+ res.append('\t(')
+ res.append(self.docname)
+ res.append(')')
+ res.append('\n')
+ return ''.join(res)
+
+ def dump(self, indent: int) -> str:
+ return ''.join([
+ self.to_string(indent),
+ *(c.dump(indent + 1) for c in self._children),
+ ])
diff --git a/sphinx/domains/index.py b/sphinx/domains/index.py
index 312617719..67cc4050a 100644
--- a/sphinx/domains/index.py
+++ b/sphinx/domains/index.py
@@ -1,44 +1,128 @@
"""The index domain."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any, ClassVar
+
from docutils import nodes
from docutils.parsers.rst import directives
+
from sphinx import addnodes
from sphinx.domains import Domain
from sphinx.util import logging
from sphinx.util.docutils import ReferenceRole, SphinxDirective
from sphinx.util.index_entries import split_index_msg
from sphinx.util.nodes import process_index_entry
+
if TYPE_CHECKING:
from collections.abc import Iterable
+
from docutils.nodes import Node, system_message
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata, OptionSpec
+
+
logger = logging.getLogger(__name__)
class IndexDomain(Domain):
"""Index domain."""
+
name = 'index'
label = 'index'
- def process_doc(self, env: BuildEnvironment, docname: str, document: Node
- ) ->None:
+ @property
+ def entries(self) -> dict[str, list[tuple[str, str, str, str, str | None]]]:
+ return self.data.setdefault('entries', {})
+
+ def clear_doc(self, docname: str) -> None:
+ self.entries.pop(docname, None)
+
+ def merge_domaindata(self, docnames: Iterable[str], otherdata: dict[str, Any]) -> None:
+ for docname in docnames:
+ self.entries[docname] = otherdata['entries'][docname]
+
+ def process_doc(self, env: BuildEnvironment, docname: str, document: Node) -> None:
"""Process a document after it is read by the environment."""
- pass
+ entries = self.entries.setdefault(env.docname, [])
+ for node in list(document.findall(addnodes.index)):
+ try:
+ for (entry_type, value, _target_id, _main, _category_key) in node['entries']:
+ split_index_msg(entry_type, value)
+ except ValueError as exc:
+ logger.warning(str(exc), location=node)
+ node.parent.remove(node)
+ else:
+ for entry in node['entries']:
+ entries.append(entry)
class IndexDirective(SphinxDirective):
"""
Directive to add entries to the index.
"""
+
has_content = False
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
- option_spec: ClassVar[OptionSpec] = {'name': directives.unchanged}
+ option_spec: ClassVar[OptionSpec] = {
+ 'name': directives.unchanged,
+ }
+
+ def run(self) -> list[Node]:
+ arguments = self.arguments[0].split('\n')
+
+ if 'name' in self.options:
+ targetname = self.options['name']
+ targetnode = nodes.target('', '', names=[targetname])
+ else:
+ targetid = 'index-%s' % self.env.new_serialno('index')
+ targetnode = nodes.target('', '', ids=[targetid])
+
+ self.state.document.note_explicit_target(targetnode)
+ indexnode = addnodes.index()
+ indexnode['entries'] = []
+ indexnode['inline'] = False
+ self.set_source_info(indexnode)
+ for entry in arguments:
+ indexnode['entries'].extend(process_index_entry(entry, targetnode['ids'][0]))
+ return [indexnode, targetnode]
class IndexRole(ReferenceRole):
- pass
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ target_id = 'index-%s' % self.env.new_serialno('index')
+ if self.has_explicit_title:
+ # if an explicit target is given, process it as a full entry
+ title = self.title
+ entries = process_index_entry(self.target, target_id)
+ else:
+ # otherwise we just create a single entry
+ if self.target.startswith('!'):
+ title = self.title[1:]
+ entries = [('single', self.target[1:], target_id, 'main', None)]
+ else:
+ title = self.title
+ entries = [('single', self.target, target_id, '', None)]
+
+ index = addnodes.index(entries=entries)
+ target = nodes.target('', '', ids=[target_id])
+ text = nodes.Text(title)
+ self.set_source_info(index)
+ return [index, target, text], []
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_domain(IndexDomain)
+ app.add_directive('index', IndexDirective)
+ app.add_role('index', IndexRole())
+
+ return {
+ 'version': 'builtin',
+ 'env_version': 1,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/domains/javascript.py b/sphinx/domains/javascript.py
index 760e3b217..540b7a6c6 100644
--- a/sphinx/domains/javascript.py
+++ b/sphinx/domains/javascript.py
@@ -1,9 +1,13 @@
"""The JavaScript domain."""
+
from __future__ import annotations
+
import contextlib
from typing import TYPE_CHECKING, Any, ClassVar, cast
+
from docutils import nodes
from docutils.parsers.rst import directives
+
from sphinx import addnodes
from sphinx.directives import ObjectDescription
from sphinx.domains import Domain, ObjType
@@ -14,14 +18,18 @@ from sphinx.util import logging
from sphinx.util.docfields import Field, GroupedField, TypedField
from sphinx.util.docutils import SphinxDirective
from sphinx.util.nodes import make_id, make_refnode
+
if TYPE_CHECKING:
from collections.abc import Iterator
+
from docutils.nodes import Element, Node
+
from sphinx.addnodes import desc_signature, pending_xref
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata, OptionSpec
+
logger = logging.getLogger(__name__)
@@ -29,25 +37,142 @@ class JSObject(ObjectDescription[tuple[str, str]]):
"""
Description of a JavaScript object.
"""
+
+ #: If set to ``True`` this object is callable and a `desc_parameterlist` is
+ #: added
has_arguments = False
+
+ #: If ``allow_nesting`` is ``True``, the object prefixes will be accumulated
+ #: based on directive nesting
allow_nesting = False
- option_spec: ClassVar[OptionSpec] = {'no-index': directives.flag,
- 'no-index-entry': directives.flag, 'no-contents-entry': directives.
- flag, 'no-typesetting': directives.flag, 'noindex': directives.flag,
- 'noindexentry': directives.flag, 'nocontentsentry': directives.flag,
- 'single-line-parameter-list': directives.flag}
-
- def handle_signature(self, sig: str, signode: desc_signature) ->tuple[
- str, str]:
+
+ option_spec: ClassVar[OptionSpec] = {
+ 'no-index': directives.flag,
+ 'no-index-entry': directives.flag,
+ 'no-contents-entry': directives.flag,
+ 'no-typesetting': directives.flag,
+ 'noindex': directives.flag,
+ 'noindexentry': directives.flag,
+ 'nocontentsentry': directives.flag,
+ 'single-line-parameter-list': directives.flag,
+ }
+
+ def get_display_prefix(self) -> list[Node]:
+ #: what is displayed right before the documentation entry
+ return []
+
+ def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
"""Breaks down construct signatures
Parses out prefix and argument list from construct definition. The
namespace and class will be determined by the nesting of domain
directives.
"""
- pass
+ sig = sig.strip()
+ if '(' in sig and sig[-1:] == ')':
+ member, arglist = sig.split('(', 1)
+ member = member.strip()
+ arglist = arglist[:-1].strip()
+ else:
+ member = sig
+ arglist = None
+ # If construct is nested, prefix the current prefix
+ prefix = self.env.ref_context.get('js:object', None)
+ mod_name = self.env.ref_context.get('js:module')
+
+ name = member
+ try:
+ member_prefix, member_name = member.rsplit('.', 1)
+ except ValueError:
+ member_name = name
+ member_prefix = ''
+ finally:
+ name = member_name
+ if prefix and member_prefix:
+ prefix = f'{prefix}.{member_prefix}'
+ elif prefix is None and member_prefix:
+ prefix = member_prefix
+ fullname = name
+ if prefix:
+ fullname = f'{prefix}.{name}'
- def before_content(self) ->None:
+ signode['module'] = mod_name
+ signode['object'] = prefix
+ signode['fullname'] = fullname
+
+ max_len = (self.env.config.javascript_maximum_signature_line_length
+ or self.env.config.maximum_signature_line_length
+ or 0)
+ multi_line_parameter_list = (
+ 'single-line-parameter-list' not in self.options
+ and (len(sig) > max_len > 0)
+ )
+
+ display_prefix = self.get_display_prefix()
+ if display_prefix:
+ signode += addnodes.desc_annotation('', '', *display_prefix)
+
+ actual_prefix = None
+ if prefix:
+ actual_prefix = prefix
+ elif mod_name:
+ actual_prefix = mod_name
+ if actual_prefix:
+ addName = addnodes.desc_addname('', '')
+ for p in actual_prefix.split('.'):
+ addName += addnodes.desc_sig_name(p, p)
+ addName += addnodes.desc_sig_punctuation('.', '.')
+ signode += addName
+ signode += addnodes.desc_name('', '', addnodes.desc_sig_name(name, name))
+ if self.has_arguments:
+ if not arglist:
+ signode += addnodes.desc_parameterlist()
+ else:
+ _pseudo_parse_arglist(signode, arglist, multi_line_parameter_list)
+ return fullname, prefix
+
+ def _object_hierarchy_parts(self, sig_node: desc_signature) -> tuple[str, ...]:
+ if 'fullname' not in sig_node:
+ return ()
+ modname = sig_node.get('module')
+ fullname = sig_node['fullname']
+
+ if modname:
+ return (modname, *fullname.split('.'))
+ else:
+ return tuple(fullname.split('.'))
+
+ def add_target_and_index(self, name_obj: tuple[str, str], sig: str,
+ signode: desc_signature) -> None:
+ mod_name = self.env.ref_context.get('js:module')
+ fullname = (mod_name + '.' if mod_name else '') + name_obj[0]
+ node_id = make_id(self.env, self.state.document, '', fullname)
+ signode['ids'].append(node_id)
+ self.state.document.note_explicit_target(signode)
+
+ domain = cast(JavaScriptDomain, self.env.get_domain('js'))
+ domain.note_object(fullname, self.objtype, node_id, location=signode)
+
+ if 'no-index-entry' not in self.options:
+ indextext = self.get_index_text(mod_name, name_obj) # type: ignore[arg-type]
+ if indextext:
+ self.indexnode['entries'].append(('single', indextext, node_id, '', None))
+
+ def get_index_text(self, objectname: str, name_obj: tuple[str, str]) -> str:
+ name, obj = name_obj
+ if self.objtype == 'function':
+ if not obj:
+ return _('%s() (built-in function)') % name
+ return _('%s() (%s method)') % (name, obj)
+ elif self.objtype == 'class':
+ return _('%s() (class)') % name
+ elif self.objtype == 'data':
+ return _('%s (global variable or constant)') % name
+ elif self.objtype == 'attribute':
+ return _('%s (%s attribute)') % (name, obj)
+ return ''
+
+ def before_content(self) -> None:
"""Handle object nesting before content
:py:class:`JSObject` represents JavaScript language constructs. For
@@ -71,9 +196,19 @@ class JSObject(ObjectDescription[tuple[str, str]]):
Current object prefix. This should generally reflect the last
element in the prefix history
"""
- pass
+ prefix = None
+ if self.names:
+ (obj_name, obj_name_prefix) = self.names.pop()
+ prefix = obj_name_prefix.strip('.') if obj_name_prefix else None
+ if self.allow_nesting:
+ prefix = obj_name
+ if prefix:
+ self.env.ref_context['js:object'] = prefix
+ if self.allow_nesting:
+ objects = self.env.ref_context.setdefault('js:objects', [])
+ objects.append(prefix)
- def after_content(self) ->None:
+ def after_content(self) -> None:
"""Handle object de-nesting after content
If this class is a nestable object, removing the last nested class prefix
@@ -83,25 +218,62 @@ class JSObject(ObjectDescription[tuple[str, str]]):
be altered as we didn't affect the nesting levels in
:py:meth:`before_content`.
"""
- pass
+ objects = self.env.ref_context.setdefault('js:objects', [])
+ if self.allow_nesting:
+ with contextlib.suppress(IndexError):
+ objects.pop()
+
+ self.env.ref_context['js:object'] = (objects[-1] if len(objects) > 0
+ else None)
+
+ def _toc_entry_name(self, sig_node: desc_signature) -> str:
+ if not sig_node.get('_toc_parts'):
+ return ''
+
+ config = self.env.app.config
+ objtype = sig_node.parent.get('objtype')
+ if config.add_function_parentheses and objtype in {'function', 'method'}:
+ parens = '()'
+ else:
+ parens = ''
+ *parents, name = sig_node['_toc_parts']
+ if config.toc_object_entries_show_parents == 'domain':
+ return sig_node.get('fullname', name) + parens
+ if config.toc_object_entries_show_parents == 'hide':
+ return name + parens
+ if config.toc_object_entries_show_parents == 'all':
+ return '.'.join([*parents, name + parens])
+ return ''
class JSCallable(JSObject):
"""Description of a JavaScript function, method or constructor."""
+
has_arguments = True
- doc_field_types = [TypedField('arguments', label=_('Arguments'), names=
- ('argument', 'arg', 'parameter', 'param'), typerolename='func',
- typenames=('paramtype', 'type')), GroupedField('errors', label=_(
- 'Throws'), rolename='func', names=('throws',), can_collapse=True),
- Field('returnvalue', label=_('Returns'), has_arg=False, names=(
- 'returns', 'return')), Field('returntype', label=_('Return type'),
- has_arg=False, names=('rtype',))]
+
+ doc_field_types = [
+ TypedField('arguments', label=_('Arguments'),
+ names=('argument', 'arg', 'parameter', 'param'),
+ typerolename='func', typenames=('paramtype', 'type')),
+ GroupedField('errors', label=_('Throws'), rolename='func',
+ names=('throws', ),
+ can_collapse=True),
+ Field('returnvalue', label=_('Returns'), has_arg=False,
+ names=('returns', 'return')),
+ Field('returntype', label=_('Return type'), has_arg=False,
+ names=('rtype',)),
+ ]
class JSConstructor(JSCallable):
"""Like a callable but with a different prefix."""
+
allow_nesting = True
+ def get_display_prefix(self) -> list[Node]:
+ return [addnodes.desc_sig_keyword('class', 'class'),
+ addnodes.desc_sig_space()]
+
class JSModule(SphinxDirective):
"""
@@ -121,32 +293,216 @@ class JSModule(SphinxDirective):
:param mod_name: Module name
"""
+
has_content = True
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = False
- option_spec: ClassVar[OptionSpec] = {'no-index': directives.flag,
- 'no-contents-entry': directives.flag, 'no-typesetting': directives.
- flag, 'noindex': directives.flag, 'nocontentsentry': directives.flag}
+ option_spec: ClassVar[OptionSpec] = {
+ 'no-index': directives.flag,
+ 'no-contents-entry': directives.flag,
+ 'no-typesetting': directives.flag,
+ 'noindex': directives.flag,
+ 'nocontentsentry': directives.flag,
+ }
+
+ def run(self) -> list[Node]:
+ mod_name = self.arguments[0].strip()
+ self.env.ref_context['js:module'] = mod_name
+ no_index = 'no-index' in self.options
+
+ content_nodes = self.parse_content_to_nodes(allow_section_headings=True)
+
+ ret: list[Node] = []
+ if not no_index:
+ domain = cast(JavaScriptDomain, self.env.get_domain('js'))
+
+ node_id = make_id(self.env, self.state.document, 'module', mod_name)
+ domain.note_module(mod_name, node_id)
+ # Make a duplicate entry in 'objects' to facilitate searching for
+ # the module in JavaScriptDomain.find_obj()
+ domain.note_object(mod_name, 'module', node_id,
+ location=(self.env.docname, self.lineno))
+
+ # The node order is: index node first, then target node
+ indextext = _('%s (module)') % mod_name
+ inode = addnodes.index(entries=[('single', indextext, node_id, '', None)])
+ ret.append(inode)
+ target = nodes.target('', '', ids=[node_id], ismod=True)
+ self.state.document.note_explicit_target(target)
+ ret.append(target)
+ ret.extend(content_nodes)
+ return ret
class JSXRefRole(XRefRole):
- pass
+ def process_link(self, env: BuildEnvironment, refnode: Element,
+ has_explicit_title: bool, title: str, target: str) -> tuple[str, str]:
+ # basically what sphinx.domains.python.PyXRefRole does
+ refnode['js:object'] = env.ref_context.get('js:object')
+ refnode['js:module'] = env.ref_context.get('js:module')
+ if not has_explicit_title:
+ title = title.lstrip('.')
+ target = target.lstrip('~')
+ if title[0:1] == '~':
+ title = title[1:]
+ dot = title.rfind('.')
+ if dot != -1:
+ title = title[dot + 1:]
+ if target[0:1] == '.':
+ target = target[1:]
+ refnode['refspecific'] = True
+ return title, target
class JavaScriptDomain(Domain):
"""JavaScript language domain."""
+
name = 'js'
label = 'JavaScript'
- object_types = {'function': ObjType(_('function'), 'func'), 'method':
- ObjType(_('method'), 'meth'), 'class': ObjType(_('class'), 'class'),
- 'data': ObjType(_('data'), 'data'), 'attribute': ObjType(_(
- 'attribute'), 'attr'), 'module': ObjType(_('module'), 'mod')}
- directives = {'function': JSCallable, 'method': JSCallable, 'class':
- JSConstructor, 'data': JSObject, 'attribute': JSObject, 'module':
- JSModule}
- roles = {'func': JSXRefRole(fix_parens=True), 'meth': JSXRefRole(
- fix_parens=True), 'class': JSXRefRole(fix_parens=True), 'data':
- JSXRefRole(), 'attr': JSXRefRole(), 'mod': JSXRefRole()}
- initial_data: dict[str, dict[str, tuple[str, str]]] = {'objects': {},
- 'modules': {}}
+ # if you add a new object type make sure to edit JSObject.get_index_string
+ object_types = {
+ 'function': ObjType(_('function'), 'func'),
+ 'method': ObjType(_('method'), 'meth'),
+ 'class': ObjType(_('class'), 'class'),
+ 'data': ObjType(_('data'), 'data'),
+ 'attribute': ObjType(_('attribute'), 'attr'),
+ 'module': ObjType(_('module'), 'mod'),
+ }
+ directives = {
+ 'function': JSCallable,
+ 'method': JSCallable,
+ 'class': JSConstructor,
+ 'data': JSObject,
+ 'attribute': JSObject,
+ 'module': JSModule,
+ }
+ roles = {
+ 'func': JSXRefRole(fix_parens=True),
+ 'meth': JSXRefRole(fix_parens=True),
+ 'class': JSXRefRole(fix_parens=True),
+ 'data': JSXRefRole(),
+ 'attr': JSXRefRole(),
+ 'mod': JSXRefRole(),
+ }
+ initial_data: dict[str, dict[str, tuple[str, str]]] = {
+ 'objects': {}, # fullname -> docname, node_id, objtype
+ 'modules': {}, # modname -> docname, node_id
+ }
+
+ @property
+ def objects(self) -> dict[str, tuple[str, str, str]]:
+ return self.data.setdefault('objects', {}) # fullname -> docname, node_id, objtype
+
+ def note_object(self, fullname: str, objtype: str, node_id: str,
+ location: Any = None) -> None:
+ if fullname in self.objects:
+ docname = self.objects[fullname][0]
+ logger.warning(__('duplicate %s description of %s, other %s in %s'),
+ objtype, fullname, objtype, docname, location=location)
+ self.objects[fullname] = (self.env.docname, node_id, objtype)
+
+ @property
+ def modules(self) -> dict[str, tuple[str, str]]:
+ return self.data.setdefault('modules', {}) # modname -> docname, node_id
+
+ def note_module(self, modname: str, node_id: str) -> None:
+ self.modules[modname] = (self.env.docname, node_id)
+
+ def clear_doc(self, docname: str) -> None:
+ for fullname, (pkg_docname, _node_id, _l) in list(self.objects.items()):
+ if pkg_docname == docname:
+ del self.objects[fullname]
+ for modname, (pkg_docname, _node_id) in list(self.modules.items()):
+ if pkg_docname == docname:
+ del self.modules[modname]
+
+ def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> None:
+ # XXX check duplicates
+ for fullname, (fn, node_id, objtype) in otherdata['objects'].items():
+ if fn in docnames:
+ self.objects[fullname] = (fn, node_id, objtype)
+ for mod_name, (pkg_docname, node_id) in otherdata['modules'].items():
+ if pkg_docname in docnames:
+ self.modules[mod_name] = (pkg_docname, node_id)
+
+ def find_obj(
+ self,
+ env: BuildEnvironment,
+ mod_name: str,
+ prefix: str,
+ name: str,
+ typ: str | None,
+ searchorder: int = 0,
+ ) -> tuple[str | None, tuple[str, str, str] | None]:
+ if name[-2:] == '()':
+ name = name[:-2]
+
+ searches = []
+ if mod_name and prefix:
+ searches.append(f'{mod_name}.{prefix}.{name}')
+ if mod_name:
+ searches.append(f'{mod_name}.{name}')
+ if prefix:
+ searches.append(f'{prefix}.{name}')
+ searches.append(name)
+
+ if searchorder == 0:
+ searches.reverse()
+
+ newname = None
+ object_ = None
+ for search_name in searches:
+ if search_name in self.objects:
+ newname = search_name
+ object_ = self.objects[search_name]
+
+ return newname, object_
+
+ def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
+ typ: str, target: str, node: pending_xref, contnode: Element,
+ ) -> Element | None:
+ mod_name = node.get('js:module')
+ prefix = node.get('js:object')
+ searchorder = 1 if node.hasattr('refspecific') else 0
+ name, obj = self.find_obj(env, mod_name, prefix, target, typ, searchorder)
+ if not obj:
+ return None
+ return make_refnode(builder, fromdocname, obj[0], obj[1], contnode, name)
+
+ def resolve_any_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
+ target: str, node: pending_xref, contnode: Element,
+ ) -> list[tuple[str, Element]]:
+ mod_name = node.get('js:module')
+ prefix = node.get('js:object')
+ name, obj = self.find_obj(env, mod_name, prefix, target, None, 1)
+ if not obj:
+ return []
+ return [('js:' + self.role_for_objtype(obj[2]), # type: ignore[operator]
+ make_refnode(builder, fromdocname, obj[0], obj[1], contnode, name))]
+
+ def get_objects(self) -> Iterator[tuple[str, str, str, str, str, int]]:
+ for refname, (docname, node_id, typ) in list(self.objects.items()):
+ yield refname, refname, typ, docname, node_id, 1
+
+ def get_full_qualified_name(self, node: Element) -> str | None:
+ modname = node.get('js:module')
+ prefix = node.get('js:object')
+ target = node.get('reftarget')
+ if target is None:
+ return None
+ else:
+ return '.'.join(filter(None, [modname, prefix, target]))
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_domain(JavaScriptDomain)
+ app.add_config_value(
+ 'javascript_maximum_signature_line_length', None, 'env', {int, type(None)},
+ )
+ return {
+ 'version': 'builtin',
+ 'env_version': 3,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/domains/math.py b/sphinx/domains/math.py
index 18f76a695..afa36062d 100644
--- a/sphinx/domains/math.py
+++ b/sphinx/domains/math.py
@@ -1,32 +1,158 @@
"""The math domain."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any
+
from docutils import nodes
from docutils.nodes import Element, Node, make_id, system_message
+
from sphinx.domains import Domain
from sphinx.locale import __
from sphinx.roles import XRefRole
from sphinx.util import logging
from sphinx.util.nodes import make_refnode
+
if TYPE_CHECKING:
from collections.abc import Iterable
+
from sphinx.addnodes import pending_xref
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata
+
+
logger = logging.getLogger(__name__)
class MathReferenceRole(XRefRole):
- pass
+ def result_nodes(self, document: nodes.document, env: BuildEnvironment, node: Element,
+ is_ref: bool) -> tuple[list[Node], list[system_message]]:
+ node['refdomain'] = 'math'
+ return [node], []
class MathDomain(Domain):
"""Mathematics domain."""
+
name = 'math'
label = 'mathematics'
- initial_data: dict[str, Any] = {'objects': {}, 'has_equations': {}}
- dangling_warnings = {'eq': 'equation not found: %(target)s'}
- enumerable_nodes = {nodes.math_block: ('displaymath', None)}
- roles = {'numref': MathReferenceRole()}
+
+ initial_data: dict[str, Any] = {
+ 'objects': {}, # labelid -> (docname, eqno)
+ 'has_equations': {}, # docname -> bool
+ }
+ dangling_warnings = {
+ 'eq': 'equation not found: %(target)s',
+ }
+ enumerable_nodes = { # node_class -> (figtype, title_getter)
+ nodes.math_block: ('displaymath', None),
+ }
+ roles = {
+ 'numref': MathReferenceRole(),
+ }
+
+ @property
+ def equations(self) -> dict[str, tuple[str, int]]:
+ return self.data.setdefault('objects', {}) # labelid -> (docname, eqno)
+
+ def note_equation(self, docname: str, labelid: str, location: Any = None) -> None:
+ if labelid in self.equations:
+ other = self.equations[labelid][0]
+ logger.warning(__('duplicate label of equation %s, other instance in %s'),
+ labelid, other, location=location)
+
+ self.equations[labelid] = (docname, self.env.new_serialno('eqno') + 1)
+
+ def get_equation_number_for(self, labelid: str) -> int | None:
+ if labelid in self.equations:
+ return self.equations[labelid][1]
+ else:
+ return None
+
+ def process_doc(self, env: BuildEnvironment, docname: str,
+ document: nodes.document) -> None:
+ def math_node(node: Node) -> bool:
+ return isinstance(node, nodes.math | nodes.math_block)
+
+ self.data['has_equations'][docname] = any(document.findall(math_node))
+
+ def clear_doc(self, docname: str) -> None:
+ for equation_id, (doc, _eqno) in list(self.equations.items()):
+ if doc == docname:
+ del self.equations[equation_id]
+
+ self.data['has_equations'].pop(docname, None)
+
+ def merge_domaindata(self, docnames: Iterable[str], otherdata: dict[str, Any]) -> None:
+ for labelid, (doc, eqno) in otherdata['objects'].items():
+ if doc in docnames:
+ self.equations[labelid] = (doc, eqno)
+
+ for docname in docnames:
+ self.data['has_equations'][docname] = otherdata['has_equations'][docname]
+
+ def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
+ typ: str, target: str, node: pending_xref, contnode: Element,
+ ) -> Element | None:
+ assert typ in ('eq', 'numref')
+ result = self.equations.get(target)
+ if result:
+ docname, number = result
+ # TODO: perhaps use rather a sphinx-core provided prefix here?
+ node_id = make_id('equation-%s' % target)
+ if env.config.math_numfig and env.config.numfig:
+ if docname in env.toc_fignumbers:
+ numbers = env.toc_fignumbers[docname]['displaymath'].get(node_id, ())
+ eqno = '.'.join(map(str, numbers))
+ eqno = env.config.math_numsep.join(eqno.rsplit('.', 1))
+ else:
+ eqno = ''
+ else:
+ eqno = str(number)
+
+ try:
+ eqref_format = env.config.math_eqref_format or "({number})"
+ title = nodes.Text(eqref_format.format(number=eqno))
+ except KeyError as exc:
+ logger.warning(__('Invalid math_eqref_format: %r'), exc,
+ location=node)
+ title = nodes.Text("(%d)" % number)
+ title = nodes.Text("(%d)" % number)
+ return make_refnode(builder, fromdocname, docname, node_id, title)
+ else:
+ return None
+
+ def resolve_any_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
+ target: str, node: pending_xref, contnode: Element,
+ ) -> list[tuple[str, Element]]:
+ refnode = self.resolve_xref(env, fromdocname, builder, 'eq', target, node, contnode)
+ if refnode is None:
+ return []
+ else:
+ return [('eq', refnode)]
+
+ def get_objects(self) -> Iterable[tuple[str, str, str, str, str, int]]:
+ return []
+
+ def has_equations(self, docname: str | None = None) -> bool:
+ if not docname:
+ return any(self.data['has_equations'].values())
+
+ return (
+ self.data['has_equations'].get(docname, False)
+ or any(map(self.has_equations, self.env.toctree_includes.get(docname, ())))
+ )
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_domain(MathDomain)
+ app.add_role('eq', MathReferenceRole(warn_dangling=True))
+
+ return {
+ 'version': 'builtin',
+ 'env_version': 2,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/domains/python/_annotations.py b/sphinx/domains/python/_annotations.py
index bc2418143..35525f6b1 100644
--- a/sphinx/domains/python/_annotations.py
+++ b/sphinx/domains/python/_annotations.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+
import ast
import functools
import operator
@@ -6,60 +7,523 @@ import token
from collections import deque
from inspect import Parameter
from typing import TYPE_CHECKING, Any
+
from docutils import nodes
+
from sphinx import addnodes
from sphinx.addnodes import desc_signature, pending_xref, pending_xref_condition
from sphinx.pycode.parser import Token, TokenProcessor
from sphinx.util.inspect import signature_from_str
+
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
+
from docutils.nodes import Element, Node
+
from sphinx.environment import BuildEnvironment
-def parse_reftarget(reftarget: str, suppress_prefix: bool=False) ->tuple[
- str, str, str, bool]:
+def parse_reftarget(reftarget: str, suppress_prefix: bool = False,
+ ) -> tuple[str, str, str, bool]:
"""Parse a type string and return (reftype, reftarget, title, refspecific flag)"""
- pass
+ refspecific = False
+ if reftarget.startswith('.'):
+ reftarget = reftarget[1:]
+ title = reftarget
+ refspecific = True
+ elif reftarget.startswith('~'):
+ reftarget = reftarget[1:]
+ title = reftarget.split('.')[-1]
+ elif suppress_prefix:
+ title = reftarget.split('.')[-1]
+ elif reftarget.startswith('typing.'):
+ title = reftarget[7:]
+ else:
+ title = reftarget
+ if reftarget == 'None' or reftarget.startswith('typing.'):
+ # typing module provides non-class types. Obj reference is good to refer them.
+ reftype = 'obj'
+ else:
+ reftype = 'class'
-def type_to_xref(target: str, env: BuildEnvironment, *, suppress_prefix:
- bool=False) ->addnodes.pending_xref:
+ return reftype, reftarget, title, refspecific
+
+
+def type_to_xref(target: str, env: BuildEnvironment, *,
+ suppress_prefix: bool = False) -> addnodes.pending_xref:
"""Convert a type string to a cross reference node."""
- pass
+ if env:
+ kwargs = {'py:module': env.ref_context.get('py:module'),
+ 'py:class': env.ref_context.get('py:class')}
+ else:
+ kwargs = {}
+
+ reftype, target, title, refspecific = parse_reftarget(target, suppress_prefix)
+ if env.config.python_use_unqualified_type_names:
+ # Note: It would be better to use qualname to describe the object to support support
+ # nested classes. But python domain can't access the real python object because this
+ # module should work not-dynamically.
+ shortname = title.split('.')[-1]
+ contnodes: list[Node] = [pending_xref_condition('', shortname, condition='resolved'),
+ pending_xref_condition('', title, condition='*')]
+ else:
+ contnodes = [nodes.Text(title)]
-def _parse_annotation(annotation: str, env: BuildEnvironment) ->list[Node]:
+ return pending_xref('', *contnodes,
+ refdomain='py', reftype=reftype, reftarget=target,
+ refspecific=refspecific, **kwargs)
+
+
+def _parse_annotation(annotation: str, env: BuildEnvironment) -> list[Node]:
"""Parse type annotation."""
- pass
+ short_literals = env.config.python_display_short_literal_types
+ def unparse(node: ast.AST) -> list[Node]:
+ if isinstance(node, ast.Attribute):
+ return [nodes.Text(f"{unparse(node.value)[0]}.{node.attr}")]
+ if isinstance(node, ast.BinOp):
+ result: list[Node] = unparse(node.left)
+ result.extend(unparse(node.op))
+ result.extend(unparse(node.right))
+ return result
+ if isinstance(node, ast.BitOr):
+ return [addnodes.desc_sig_space(),
+ addnodes.desc_sig_punctuation('', '|'),
+ addnodes.desc_sig_space()]
+ if isinstance(node, ast.Constant):
+ if node.value is Ellipsis:
+ return [addnodes.desc_sig_punctuation('', "...")]
+ if isinstance(node.value, bool):
+ return [addnodes.desc_sig_keyword('', repr(node.value))]
+ if isinstance(node.value, int):
+ return [addnodes.desc_sig_literal_number('', repr(node.value))]
+ if isinstance(node.value, str):
+ return [addnodes.desc_sig_literal_string('', repr(node.value))]
+ else:
+ # handles None, which is further handled by type_to_xref later
+ # and fallback for other types that should be converted
+ return [nodes.Text(repr(node.value))]
+ if isinstance(node, ast.Expr):
+ return unparse(node.value)
+ if isinstance(node, ast.Invert):
+ return [addnodes.desc_sig_punctuation('', '~')]
+ if isinstance(node, ast.USub):
+ return [addnodes.desc_sig_punctuation('', '-')]
+ if isinstance(node, ast.List):
+ result = [addnodes.desc_sig_punctuation('', '[')]
+ if node.elts:
+ # check if there are elements in node.elts to only pop the
+ # last element of result if the for-loop was run at least
+ # once
+ for elem in node.elts:
+ result.extend(unparse(elem))
+ result.append(addnodes.desc_sig_punctuation('', ','))
+ result.append(addnodes.desc_sig_space())
+ result.pop()
+ result.pop()
+ result.append(addnodes.desc_sig_punctuation('', ']'))
+ return result
+ if isinstance(node, ast.Module):
+ return functools.reduce(operator.iadd, (unparse(e) for e in node.body), [])
+ if isinstance(node, ast.Name):
+ return [nodes.Text(node.id)]
+ if isinstance(node, ast.Subscript):
+ if getattr(node.value, 'id', '') in {'Optional', 'Union'}:
+ return _unparse_pep_604_annotation(node)
+ if short_literals and getattr(node.value, 'id', '') == 'Literal':
+ return _unparse_pep_604_annotation(node)
+ result = unparse(node.value)
+ result.append(addnodes.desc_sig_punctuation('', '['))
+ result.extend(unparse(node.slice))
+ result.append(addnodes.desc_sig_punctuation('', ']'))
-class _TypeParameterListParser(TokenProcessor):
+ # Wrap the Text nodes inside brackets by literal node if the subscript is a Literal
+ if result[0] in ('Literal', 'typing.Literal'):
+ for i, subnode in enumerate(result[1:], start=1):
+ if isinstance(subnode, nodes.Text):
+ result[i] = nodes.literal('', '', subnode)
+ return result
+ if isinstance(node, ast.UnaryOp):
+ return unparse(node.op) + unparse(node.operand)
+ if isinstance(node, ast.Tuple):
+ if node.elts:
+ result = []
+ for elem in node.elts:
+ result.extend(unparse(elem))
+ result.append(addnodes.desc_sig_punctuation('', ','))
+ result.append(addnodes.desc_sig_space())
+ result.pop()
+ result.pop()
+ else:
+ result = [addnodes.desc_sig_punctuation('', '('),
+ addnodes.desc_sig_punctuation('', ')')]
- def __init__(self, sig: str) ->None:
+ return result
+ if isinstance(node, ast.Call):
+ # Call nodes can be used in Annotated type metadata,
+ # for example Annotated[str, ArbitraryTypeValidator(str, len=10)]
+ args = []
+ for arg in node.args:
+ args += unparse(arg)
+ args.append(addnodes.desc_sig_punctuation('', ','))
+ args.append(addnodes.desc_sig_space())
+ for kwd in node.keywords:
+ args.append(addnodes.desc_sig_name(kwd.arg, kwd.arg)) # type: ignore[arg-type]
+ args.append(addnodes.desc_sig_operator('', '='))
+ args += unparse(kwd.value)
+ args.append(addnodes.desc_sig_punctuation('', ','))
+ args.append(addnodes.desc_sig_space())
+ result = [
+ *unparse(node.func),
+ addnodes.desc_sig_punctuation('', '('),
+ *args[:-2], # skip the final comma and space
+ addnodes.desc_sig_punctuation('', ')'),
+ ]
+ return result
+ msg = f'unsupported syntax: {node}'
+ raise SyntaxError(msg) # unsupported syntax
+
+ def _unparse_pep_604_annotation(node: ast.Subscript) -> list[Node]:
+ subscript = node.slice
+
+ flattened: list[Node] = []
+ if isinstance(subscript, ast.Tuple):
+ flattened.extend(unparse(subscript.elts[0]))
+ for elt in subscript.elts[1:]:
+ flattened.extend(unparse(ast.BitOr()))
+ flattened.extend(unparse(elt))
+ else:
+ # e.g. a Union[] inside an Optional[]
+ flattened.extend(unparse(subscript))
+
+ if getattr(node.value, 'id', '') == 'Optional':
+ flattened.extend(unparse(ast.BitOr()))
+ flattened.append(nodes.Text('None'))
+
+ return flattened
+
+ try:
+ tree = ast.parse(annotation, type_comments=True)
+ result: list[Node] = []
+ for node in unparse(tree):
+ if isinstance(node, nodes.literal):
+ result.append(node[0])
+ elif isinstance(node, nodes.Text) and node.strip():
+ if (result and isinstance(result[-1], addnodes.desc_sig_punctuation) and
+ result[-1].astext() == '~'):
+ result.pop()
+ result.append(type_to_xref(str(node), env, suppress_prefix=True))
+ else:
+ result.append(type_to_xref(str(node), env))
+ else:
+ result.append(node)
+ return result
+ except SyntaxError:
+ return [type_to_xref(annotation, env)]
+
+
+class _TypeParameterListParser(TokenProcessor):
+ def __init__(self, sig: str) -> None:
signature = sig.replace('\n', '').strip()
super().__init__([signature])
+ # Each item is a tuple (name, kind, default, annotation) mimicking
+ # ``inspect.Parameter`` to allow default values on VAR_POSITIONAL
+ # or VAR_KEYWORD parameters.
self.type_params: list[tuple[str, int, Any, Any]] = []
+ def fetch_type_param_spec(self) -> list[Token]:
+ tokens = []
+ while current := self.fetch_token():
+ tokens.append(current)
+ for ldelim, rdelim in ('(', ')'), ('{', '}'), ('[', ']'):
+ if current == [token.OP, ldelim]:
+ tokens += self.fetch_until([token.OP, rdelim])
+ break
+ else:
+ if current == token.INDENT:
+ tokens += self.fetch_until(token.DEDENT)
+ elif current.match(
+ [token.OP, ':'], [token.OP, '='], [token.OP, ',']):
+ tokens.pop()
+ break
+ return tokens
+
+ def parse(self) -> None:
+ while current := self.fetch_token():
+ if current == token.NAME:
+ tp_name = current.value.strip()
+ if self.previous and self.previous.match([token.OP, '*'], [token.OP, '**']):
+ if self.previous == [token.OP, '*']:
+ tp_kind = Parameter.VAR_POSITIONAL
+ else:
+ tp_kind = Parameter.VAR_KEYWORD # type: ignore[assignment]
+ else:
+ tp_kind = Parameter.POSITIONAL_OR_KEYWORD # type: ignore[assignment]
+
+ tp_ann: Any = Parameter.empty
+ tp_default: Any = Parameter.empty
+
+ current = self.fetch_token()
+ if current and current.match([token.OP, ':'], [token.OP, '=']):
+ if current == [token.OP, ':']:
+ tokens = self.fetch_type_param_spec()
+ tp_ann = self._build_identifier(tokens)
+
+ if self.current and self.current == [token.OP, '=']:
+ tokens = self.fetch_type_param_spec()
+ tp_default = self._build_identifier(tokens)
-def _parse_type_list(tp_list: str, env: BuildEnvironment,
- multi_line_parameter_list: bool=False) ->addnodes.desc_type_parameter_list:
+ if tp_kind != Parameter.POSITIONAL_OR_KEYWORD and tp_ann != Parameter.empty:
+ msg = ('type parameter bound or constraint is not allowed '
+ f'for {tp_kind.description} parameters')
+ raise SyntaxError(msg)
+
+ type_param = (tp_name, tp_kind, tp_default, tp_ann)
+ self.type_params.append(type_param)
+
+ def _build_identifier(self, tokens: list[Token]) -> str:
+ from itertools import chain, islice
+
+ def triplewise(iterable: Iterable[Token]) -> Iterator[tuple[Token, ...]]:
+ # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG
+ it = iter(iterable)
+ window = deque(islice(it, 3), maxlen=3)
+ if len(window) == 3:
+ yield tuple(window)
+ for x in it:
+ window.append(x)
+ yield tuple(window)
+
+ idents: list[str] = []
+ tokens: Iterable[Token] = iter(tokens) # type: ignore[no-redef]
+ # do not format opening brackets
+ for tok in tokens:
+ if not tok.match([token.OP, '('], [token.OP, '['], [token.OP, '{']):
+ # check if the first non-delimiter character is an unpack operator
+ is_unpack_operator = tok.match([token.OP, '*'], [token.OP, ['**']])
+ idents.append(self._pformat_token(tok, native=is_unpack_operator))
+ break
+ idents.append(tok.value)
+
+ # check the remaining tokens
+ stop = Token(token.ENDMARKER, '', (-1, -1), (-1, -1), '<sentinel>')
+ is_unpack_operator = False
+ for tok, op, after in triplewise(chain(tokens, [stop, stop])):
+ ident = self._pformat_token(tok, native=is_unpack_operator)
+ idents.append(ident)
+ # determine if the next token is an unpack operator depending
+ # on the left and right hand side of the operator symbol
+ is_unpack_operator = (
+ op.match([token.OP, '*'], [token.OP, '**']) and not (
+ tok.match(token.NAME, token.NUMBER, token.STRING,
+ [token.OP, ')'], [token.OP, ']'], [token.OP, '}'])
+ and after.match(token.NAME, token.NUMBER, token.STRING,
+ [token.OP, '('], [token.OP, '['], [token.OP, '{'])
+ )
+ )
+
+ return ''.join(idents).strip()
+
+ def _pformat_token(self, tok: Token, native: bool = False) -> str:
+ if native:
+ return tok.value
+
+ if tok.match(token.NEWLINE, token.ENDMARKER):
+ return ''
+
+ if tok.match([token.OP, ':'], [token.OP, ','], [token.OP, '#']):
+ return f'{tok.value} '
+
+ # Arithmetic operators are allowed because PEP 695 specifies the
+ # default type parameter to be *any* expression (so "T1 << T2" is
+ # allowed if it makes sense). The caller is responsible to ensure
+ # that a multiplication operator ("*") is not to be confused with
+ # an unpack operator (which will not be surrounded by spaces).
+ #
+ # The operators are ordered according to how likely they are to
+ # be used and for (possible) future implementations (e.g., "&" for
+ # an intersection type).
+ if tok.match(
+ # Most likely operators to appear
+ [token.OP, '='], [token.OP, '|'],
+ # Type composition (future compatibility)
+ [token.OP, '&'], [token.OP, '^'], [token.OP, '<'], [token.OP, '>'],
+ # Unlikely type composition
+ [token.OP, '+'], [token.OP, '-'], [token.OP, '*'], [token.OP, '**'],
+ # Unlikely operators but included for completeness
+ [token.OP, '@'], [token.OP, '/'], [token.OP, '//'], [token.OP, '%'],
+ [token.OP, '<<'], [token.OP, '>>'], [token.OP, '>>>'],
+ [token.OP, '<='], [token.OP, '>='], [token.OP, '=='], [token.OP, '!='],
+ ):
+ return f' {tok.value} '
+
+ return tok.value
+
+
+def _parse_type_list(
+ tp_list: str, env: BuildEnvironment,
+ multi_line_parameter_list: bool = False,
+) -> addnodes.desc_type_parameter_list:
"""Parse a list of type parameters according to PEP 695."""
- pass
+ type_params = addnodes.desc_type_parameter_list(tp_list)
+ type_params['multi_line_parameter_list'] = multi_line_parameter_list
+ # formal parameter names are interpreted as type parameter names and
+ # type annotations are interpreted as type parameter bound or constraints
+ parser = _TypeParameterListParser(tp_list)
+ parser.parse()
+ for (tp_name, tp_kind, tp_default, tp_ann) in parser.type_params:
+ # no positional-only or keyword-only allowed in a type parameters list
+ if tp_kind in {Parameter.POSITIONAL_ONLY, Parameter.KEYWORD_ONLY}:
+ msg = ('positional-only or keyword-only parameters '
+ 'are prohibited in type parameter lists')
+ raise SyntaxError(msg)
+
+ node = addnodes.desc_type_parameter()
+ if tp_kind == Parameter.VAR_POSITIONAL:
+ node += addnodes.desc_sig_operator('', '*')
+ elif tp_kind == Parameter.VAR_KEYWORD:
+ node += addnodes.desc_sig_operator('', '**')
+ node += addnodes.desc_sig_name('', tp_name)
+
+ if tp_ann is not Parameter.empty:
+ annotation = _parse_annotation(tp_ann, env)
+ if not annotation:
+ continue
+
+ node += addnodes.desc_sig_punctuation('', ':')
+ node += addnodes.desc_sig_space()
+
+ type_ann_expr = addnodes.desc_sig_name('', '',
+ *annotation) # type: ignore[arg-type]
+ # a type bound is ``T: U`` whereas type constraints
+ # must be enclosed with parentheses. ``T: (U, V)``
+ if tp_ann.startswith('(') and tp_ann.endswith(')'):
+ type_ann_text = type_ann_expr.astext()
+ if type_ann_text.startswith('(') and type_ann_text.endswith(')'):
+ node += type_ann_expr
+ else:
+ # surrounding braces are lost when using _parse_annotation()
+ node += addnodes.desc_sig_punctuation('', '(')
+ node += type_ann_expr # type constraint
+ node += addnodes.desc_sig_punctuation('', ')')
+ else:
+ node += type_ann_expr # type bound
+
+ if tp_default is not Parameter.empty:
+ # Always surround '=' with spaces, even if there is no annotation
+ node += addnodes.desc_sig_space()
+ node += addnodes.desc_sig_operator('', '=')
+ node += addnodes.desc_sig_space()
+ node += nodes.inline('', tp_default,
+ classes=['default_value'],
+ support_smartquotes=False)
+ type_params += node
+ return type_params
-def _parse_arglist(arglist: str, env: BuildEnvironment,
- multi_line_parameter_list: bool=False) ->addnodes.desc_parameterlist:
+
+def _parse_arglist(
+ arglist: str, env: BuildEnvironment, multi_line_parameter_list: bool = False,
+) -> addnodes.desc_parameterlist:
"""Parse a list of arguments using AST parser"""
- pass
+ params = addnodes.desc_parameterlist(arglist)
+ params['multi_line_parameter_list'] = multi_line_parameter_list
+ sig = signature_from_str('(%s)' % arglist)
+ last_kind = None
+ for param in sig.parameters.values():
+ if param.kind != param.POSITIONAL_ONLY and last_kind == param.POSITIONAL_ONLY:
+ # PEP-570: Separator for Positional Only Parameter: /
+ params += addnodes.desc_parameter('', '', addnodes.desc_sig_operator('', '/'))
+ if param.kind == param.KEYWORD_ONLY and last_kind in (param.POSITIONAL_OR_KEYWORD,
+ param.POSITIONAL_ONLY,
+ None):
+ # PEP-3102: Separator for Keyword Only Parameter: *
+ params += addnodes.desc_parameter('', '', addnodes.desc_sig_operator('', '*'))
+
+ node = addnodes.desc_parameter()
+ if param.kind == param.VAR_POSITIONAL:
+ node += addnodes.desc_sig_operator('', '*')
+ node += addnodes.desc_sig_name('', param.name)
+ elif param.kind == param.VAR_KEYWORD:
+ node += addnodes.desc_sig_operator('', '**')
+ node += addnodes.desc_sig_name('', param.name)
+ else:
+ node += addnodes.desc_sig_name('', param.name)
+
+ if param.annotation is not param.empty:
+ children = _parse_annotation(param.annotation, env)
+ node += addnodes.desc_sig_punctuation('', ':')
+ node += addnodes.desc_sig_space()
+ node += addnodes.desc_sig_name('', '', *children) # type: ignore[arg-type]
+ if param.default is not param.empty:
+ if param.annotation is not param.empty:
+ node += addnodes.desc_sig_space()
+ node += addnodes.desc_sig_operator('', '=')
+ node += addnodes.desc_sig_space()
+ else:
+ node += addnodes.desc_sig_operator('', '=')
+ node += nodes.inline('', param.default, classes=['default_value'],
+ support_smartquotes=False)
+
+ params += node
+ last_kind = param.kind
+
+ if last_kind == Parameter.POSITIONAL_ONLY:
+ # PEP-570: Separator for Positional Only Parameter: /
+ params += addnodes.desc_parameter('', '', addnodes.desc_sig_operator('', '/'))
+
+ return params
-def _pseudo_parse_arglist(signode: desc_signature, arglist: str,
- multi_line_parameter_list: bool=False) ->None:
+def _pseudo_parse_arglist(
+ signode: desc_signature, arglist: str, multi_line_parameter_list: bool = False,
+) -> None:
""""Parse" a list of arguments separated by commas.
Arguments can have "optional" annotations given by enclosing them in
brackets. Currently, this will split at any comma, even if it's inside a
string literal (e.g. default argument value).
"""
- pass
+ paramlist = addnodes.desc_parameterlist()
+ paramlist['multi_line_parameter_list'] = multi_line_parameter_list
+ stack: list[Element] = [paramlist]
+ try:
+ for argument in arglist.split(','):
+ argument = argument.strip()
+ ends_open = ends_close = 0
+ while argument.startswith('['):
+ stack.append(addnodes.desc_optional())
+ stack[-2] += stack[-1]
+ argument = argument[1:].strip()
+ while argument.startswith(']'):
+ stack.pop()
+ argument = argument[1:].strip()
+ while argument.endswith(']') and not argument.endswith('[]'):
+ ends_close += 1
+ argument = argument[:-1].strip()
+ while argument.endswith('['):
+ ends_open += 1
+ argument = argument[:-1].strip()
+ if argument:
+ stack[-1] += addnodes.desc_parameter(
+ '', '', addnodes.desc_sig_name(argument, argument))
+ while ends_open:
+ stack.append(addnodes.desc_optional())
+ stack[-2] += stack[-1]
+ ends_open -= 1
+ while ends_close:
+ stack.pop()
+ ends_close -= 1
+ if len(stack) != 1:
+ raise IndexError
+ except IndexError:
+ # if there are too few or too many elements on the stack, just give up
+ # and treat the whole argument list as one argument, discarding the
+ # already partially populated paramlist node
+ paramlist = addnodes.desc_parameterlist()
+ paramlist += addnodes.desc_parameter(arglist, arglist)
+ signode += paramlist
+ else:
+ signode += paramlist
diff --git a/sphinx/domains/python/_object.py b/sphinx/domains/python/_object.py
index 317998cba..bdd1fdc97 100644
--- a/sphinx/domains/python/_object.py
+++ b/sphinx/domains/python/_object.py
@@ -1,38 +1,128 @@
from __future__ import annotations
+
import contextlib
import re
from typing import TYPE_CHECKING, ClassVar
+
from docutils import nodes
from docutils.parsers.rst import directives
+
from sphinx import addnodes
from sphinx.addnodes import desc_signature, pending_xref, pending_xref_condition
from sphinx.directives import ObjectDescription
-from sphinx.domains.python._annotations import _parse_annotation, _parse_arglist, _parse_type_list, _pseudo_parse_arglist, parse_reftarget
+from sphinx.domains.python._annotations import (
+ _parse_annotation,
+ _parse_arglist,
+ _parse_type_list,
+ _pseudo_parse_arglist,
+ parse_reftarget,
+)
from sphinx.locale import _
from sphinx.util import logging
from sphinx.util.docfields import Field, GroupedField, TypedField
-from sphinx.util.nodes import make_id
+from sphinx.util.nodes import (
+ make_id,
+)
+
if TYPE_CHECKING:
from docutils.nodes import Node
from docutils.parsers.rst.states import Inliner
+
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import OptionSpec, TextlikeNode
+
logger = logging.getLogger(__name__)
+
+# REs for Python signatures
py_sig_re = re.compile(
- """^ ([\\w.]*\\.)? # class name(s)
- (\\w+) \\s* # thing name
- (?: \\[\\s*(.*)\\s*])? # optional: type parameters list
- (?: \\(\\s*(.*)\\s*\\) # optional: arguments
- (?:\\s* -> \\s* (.*))? # return annotation
+ r'''^ ([\w.]*\.)? # class name(s)
+ (\w+) \s* # thing name
+ (?: \[\s*(.*)\s*])? # optional: type parameters list
+ (?: \(\s*(.*)\s*\) # optional: arguments
+ (?:\s* -> \s* (.*))? # return annotation
)? $ # and nothing more
- """
- , re.VERBOSE)
+ ''', re.VERBOSE)
+# This override allows our inline type specifiers to behave like :class: link
+# when it comes to handling "." and "~" prefixes.
class PyXrefMixin:
+ def make_xref(
+ self,
+ rolename: str,
+ domain: str,
+ target: str,
+ innernode: type[TextlikeNode] = nodes.emphasis,
+ contnode: Node | None = None,
+ env: BuildEnvironment | None = None,
+ inliner: Inliner | None = None,
+ location: Node | None = None,
+ ) -> Node:
+ # we use inliner=None to make sure we get the old behaviour with a single
+ # pending_xref node
+ result = super().make_xref(rolename, domain, target, # type: ignore[misc]
+ innernode, contnode,
+ env, inliner=None, location=None)
+ if isinstance(result, pending_xref):
+ assert env is not None
+ result['refspecific'] = True
+ result['py:module'] = env.ref_context.get('py:module')
+ result['py:class'] = env.ref_context.get('py:class')
+
+ reftype, reftarget, reftitle, _ = parse_reftarget(target)
+ if reftarget != reftitle:
+ result['reftype'] = reftype
+ result['reftarget'] = reftarget
+
+ result.clear()
+ result += innernode(reftitle, reftitle) # type: ignore[call-arg]
+ elif env.config.python_use_unqualified_type_names:
+ children = result.children
+ result.clear()
+
+ shortname = target.split('.')[-1]
+ textnode = innernode('', shortname) # type: ignore[call-arg]
+ contnodes = [pending_xref_condition('', '', textnode, condition='resolved'),
+ pending_xref_condition('', '', *children, condition='*')]
+ result.extend(contnodes)
+
+ return result
+
_delimiters_re = re.compile(
- '(\\s*[\\[\\]\\(\\),](?:\\s*o[rf]\\s)?\\s*|\\s+o[rf]\\s+|\\s*\\|\\s*|\\.\\.\\.)'
- )
+ r'(\s*[\[\]\(\),](?:\s*o[rf]\s)?\s*|\s+o[rf]\s+|\s*\|\s*|\.\.\.)'
+ )
+
+ def make_xrefs(
+ self,
+ rolename: str,
+ domain: str,
+ target: str,
+ innernode: type[TextlikeNode] = nodes.emphasis,
+ contnode: Node | None = None,
+ env: BuildEnvironment | None = None,
+ inliner: Inliner | None = None,
+ location: Node | None = None,
+ ) -> list[Node]:
+ sub_targets = self._delimiters_re.split(target)
+
+ split_contnode = bool(contnode and contnode.astext() == target)
+
+ in_literal = False
+ results = []
+ for sub_target in filter(None, sub_targets):
+ if split_contnode:
+ contnode = nodes.Text(sub_target)
+
+ if in_literal or self._delimiters_re.match(sub_target):
+ results.append(contnode or innernode(sub_target, sub_target)) # type: ignore[call-arg]
+ else:
+ results.append(self.make_xref(rolename, domain, sub_target,
+ innernode, contnode, env, inliner, location))
+
+ if sub_target in {'Literal', 'typing.Literal', '~typing.Literal'}:
+ in_literal = True
+
+ return results
class PyField(PyXrefMixin, Field):
@@ -54,41 +144,56 @@ class PyObject(ObjectDescription[tuple[str, str]]):
:cvar allow_nesting: Class is an object that allows for nested namespaces
:vartype allow_nesting: bool
"""
- option_spec: ClassVar[OptionSpec] = {'no-index': directives.flag,
- 'no-index-entry': directives.flag, 'no-contents-entry': directives.
- flag, 'no-typesetting': directives.flag, 'noindex': directives.flag,
- 'noindexentry': directives.flag, 'nocontentsentry': directives.flag,
+
+ option_spec: ClassVar[OptionSpec] = {
+ 'no-index': directives.flag,
+ 'no-index-entry': directives.flag,
+ 'no-contents-entry': directives.flag,
+ 'no-typesetting': directives.flag,
+ 'noindex': directives.flag,
+ 'noindexentry': directives.flag,
+ 'nocontentsentry': directives.flag,
'single-line-parameter-list': directives.flag,
- 'single-line-type-parameter-list': directives.flag, 'module':
- directives.unchanged, 'canonical': directives.unchanged,
- 'annotation': directives.unchanged}
- doc_field_types = [PyTypedField('parameter', label=_('Parameters'),
- names=('param', 'parameter', 'arg', 'argument', 'keyword', 'kwarg',
- 'kwparam'), typerolename='class', typenames=('paramtype', 'type'),
- can_collapse=True), PyTypedField('variable', label=_('Variables'),
- names=('var', 'ivar', 'cvar'), typerolename='class', typenames=(
- 'vartype',), can_collapse=True), PyGroupedField('exceptions', label
- =_('Raises'), rolename='exc', names=('raises', 'raise', 'exception',
- 'except'), can_collapse=True), Field('returnvalue', label=_(
- 'Returns'), has_arg=False, names=('returns', 'return')), PyField(
- 'returntype', label=_('Return type'), has_arg=False, names=('rtype'
- ,), bodyrolename='class')]
+ 'single-line-type-parameter-list': directives.flag,
+ 'module': directives.unchanged,
+ 'canonical': directives.unchanged,
+ 'annotation': directives.unchanged,
+ }
+
+ doc_field_types = [
+ PyTypedField('parameter', label=_('Parameters'),
+ names=('param', 'parameter', 'arg', 'argument',
+ 'keyword', 'kwarg', 'kwparam'),
+ typerolename='class', typenames=('paramtype', 'type'),
+ can_collapse=True),
+ PyTypedField('variable', label=_('Variables'),
+ names=('var', 'ivar', 'cvar'),
+ typerolename='class', typenames=('vartype',),
+ can_collapse=True),
+ PyGroupedField('exceptions', label=_('Raises'), rolename='exc',
+ names=('raises', 'raise', 'exception', 'except'),
+ can_collapse=True),
+ Field('returnvalue', label=_('Returns'), has_arg=False,
+ names=('returns', 'return')),
+ PyField('returntype', label=_('Return type'), has_arg=False,
+ names=('rtype',), bodyrolename='class'),
+ ]
+
allow_nesting = False
- def get_signature_prefix(self, sig: str) ->list[nodes.Node]:
+ def get_signature_prefix(self, sig: str) -> list[nodes.Node]:
"""May return a prefix to put before the object name in the
signature.
"""
- pass
+ return []
- def needs_arglist(self) ->bool:
+ def needs_arglist(self) -> bool:
"""May return true if an empty argument list is to be generated even if
the document contains none.
"""
- pass
+ return False
- def handle_signature(self, sig: str, signode: desc_signature) ->tuple[
- str, str]:
+ def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
"""Transform a Python signature into RST nodes.
Return (fully qualified name of the thing, classname if any).
@@ -97,13 +202,155 @@ class PyObject(ObjectDescription[tuple[str, str]]):
* it is stripped from the displayed name if present
* it is added to the full name (return value) if not present
"""
- pass
+ m = py_sig_re.match(sig)
+ if m is None:
+ raise ValueError
+ prefix, name, tp_list, arglist, retann = m.groups()
+
+ # determine module and class name (if applicable), as well as full name
+ modname = self.options.get('module', self.env.ref_context.get('py:module'))
+ classname = self.env.ref_context.get('py:class')
+ if classname:
+ add_module = False
+ if prefix and (prefix == classname or
+ prefix.startswith(classname + ".")):
+ fullname = prefix + name
+ # class name is given again in the signature
+ prefix = prefix[len(classname):].lstrip('.')
+ elif prefix:
+ # class name is given in the signature, but different
+ # (shouldn't happen)
+ fullname = classname + '.' + prefix + name
+ else:
+ # class name is not given in the signature
+ fullname = classname + '.' + name
+ else:
+ add_module = True
+ if prefix:
+ classname = prefix.rstrip('.')
+ fullname = prefix + name
+ else:
+ classname = ''
+ fullname = name
- def get_index_text(self, modname: str, name: tuple[str, str]) ->str:
+ signode['module'] = modname
+ signode['class'] = classname
+ signode['fullname'] = fullname
+
+ max_len = (self.env.config.python_maximum_signature_line_length
+ or self.env.config.maximum_signature_line_length
+ or 0)
+
+ # determine if the function arguments (without its type parameters)
+ # should be formatted on a multiline or not by removing the width of
+ # the type parameters list (if any)
+ sig_len = len(sig)
+ tp_list_span = m.span(3)
+ multi_line_parameter_list = (
+ 'single-line-parameter-list' not in self.options
+ and (sig_len - (tp_list_span[1] - tp_list_span[0])) > max_len > 0
+ )
+
+ # determine whether the type parameter list must be wrapped or not
+ arglist_span = m.span(4)
+ multi_line_type_parameter_list = (
+ 'single-line-type-parameter-list' not in self.options
+ and (sig_len - (arglist_span[1] - arglist_span[0])) > max_len > 0
+ )
+
+ sig_prefix = self.get_signature_prefix(sig)
+ if sig_prefix:
+ if type(sig_prefix) is str:
+ msg = ("Python directive method get_signature_prefix()"
+ " must return a list of nodes."
+ f" Return value was '{sig_prefix}'.")
+ raise TypeError(msg)
+ signode += addnodes.desc_annotation(str(sig_prefix), '', *sig_prefix)
+
+ if prefix:
+ signode += addnodes.desc_addname(prefix, prefix)
+ elif modname and add_module and self.env.config.add_module_names:
+ nodetext = modname + '.'
+ signode += addnodes.desc_addname(nodetext, nodetext)
+
+ signode += addnodes.desc_name(name, name)
+
+ if tp_list:
+ try:
+ signode += _parse_type_list(tp_list, self.env, multi_line_type_parameter_list)
+ except Exception as exc:
+ logger.warning("could not parse tp_list (%r): %s", tp_list, exc,
+ location=signode)
+
+ if arglist:
+ try:
+ signode += _parse_arglist(arglist, self.env, multi_line_parameter_list)
+ except SyntaxError:
+ # fallback to parse arglist original parser
+ # (this may happen if the argument list is incorrectly used
+ # as a list of bases when documenting a class)
+ # it supports to represent optional arguments (ex. "func(foo [, bar])")
+ _pseudo_parse_arglist(signode, arglist, multi_line_parameter_list)
+ except (NotImplementedError, ValueError) as exc:
+ # duplicated parameter names raise ValueError and not a SyntaxError
+ logger.warning("could not parse arglist (%r): %s", arglist, exc,
+ location=signode)
+ _pseudo_parse_arglist(signode, arglist, multi_line_parameter_list)
+ else:
+ if self.needs_arglist():
+ # for callables, add an empty parameter list
+ signode += addnodes.desc_parameterlist()
+
+ if retann:
+ children = _parse_annotation(retann, self.env)
+ signode += addnodes.desc_returns(retann, '', *children)
+
+ anno = self.options.get('annotation')
+ if anno:
+ signode += addnodes.desc_annotation(' ' + anno, '',
+ addnodes.desc_sig_space(),
+ nodes.Text(anno))
+
+ return fullname, prefix
+
+ def _object_hierarchy_parts(self, sig_node: desc_signature) -> tuple[str, ...]:
+ if 'fullname' not in sig_node:
+ return ()
+ modname = sig_node.get('module')
+ fullname = sig_node['fullname']
+
+ if modname:
+ return (modname, *fullname.split('.'))
+ else:
+ return tuple(fullname.split('.'))
+
+ def get_index_text(self, modname: str, name: tuple[str, str]) -> str:
"""Return the text for the index entry of the object."""
- pass
+ msg = 'must be implemented in subclasses'
+ raise NotImplementedError(msg)
+
+ def add_target_and_index(self, name_cls: tuple[str, str], sig: str,
+ signode: desc_signature) -> None:
+ modname = self.options.get('module', self.env.ref_context.get('py:module'))
+ fullname = (modname + '.' if modname else '') + name_cls[0]
+ node_id = make_id(self.env, self.state.document, '', fullname)
+ signode['ids'].append(node_id)
+ self.state.document.note_explicit_target(signode)
+
+ domain = self.env.domains['py']
+ domain.note_object(fullname, self.objtype, node_id, location=signode)
- def before_content(self) ->None:
+ canonical_name = self.options.get('canonical')
+ if canonical_name:
+ domain.note_object(canonical_name, self.objtype, node_id, aliased=True,
+ location=signode)
+
+ if 'no-index-entry' not in self.options:
+ indextext = self.get_index_text(modname, name_cls)
+ if indextext:
+ self.indexnode['entries'].append(('single', indextext, node_id, '', None))
+
+ def before_content(self) -> None:
"""Handle object nesting before content
:py:class:`PyObject` represents Python language constructs. For
@@ -115,9 +362,28 @@ class PyObject(ObjectDescription[tuple[str, str]]):
only the most recent object is tracked. This object prefix name will be
removed with :py:meth:`after_content`.
"""
- pass
+ prefix = None
+ if self.names:
+ # fullname and name_prefix come from the `handle_signature` method.
+ # fullname represents the full object name that is constructed using
+ # object nesting and explicit prefixes. `name_prefix` is the
+ # explicit prefix given in a signature
+ (fullname, name_prefix) = self.names[-1]
+ if self.allow_nesting:
+ prefix = fullname
+ elif name_prefix:
+ prefix = name_prefix.strip('.')
+ if prefix:
+ self.env.ref_context['py:class'] = prefix
+ if self.allow_nesting:
+ classes = self.env.ref_context.setdefault('py:classes', [])
+ classes.append(prefix)
+ if 'module' in self.options:
+ modules = self.env.ref_context.setdefault('py:modules', [])
+ modules.append(self.env.ref_context.get('py:module'))
+ self.env.ref_context['py:module'] = self.options['module']
- def after_content(self) ->None:
+ def after_content(self) -> None:
"""Handle object de-nesting after content
If this class is a nestable object, removing the last nested class prefix
@@ -127,4 +393,35 @@ class PyObject(ObjectDescription[tuple[str, str]]):
be altered as we didn't affect the nesting levels in
:py:meth:`before_content`.
"""
- pass
+ classes = self.env.ref_context.setdefault('py:classes', [])
+ if self.allow_nesting:
+ with contextlib.suppress(IndexError):
+ classes.pop()
+
+ self.env.ref_context['py:class'] = (classes[-1] if len(classes) > 0
+ else None)
+ if 'module' in self.options:
+ modules = self.env.ref_context.setdefault('py:modules', [])
+ if modules:
+ self.env.ref_context['py:module'] = modules.pop()
+ else:
+ self.env.ref_context.pop('py:module')
+
+ def _toc_entry_name(self, sig_node: desc_signature) -> str:
+ if not sig_node.get('_toc_parts'):
+ return ''
+
+ config = self.env.app.config
+ objtype = sig_node.parent.get('objtype')
+ if config.add_function_parentheses and objtype in {'function', 'method'}:
+ parens = '()'
+ else:
+ parens = ''
+ *parents, name = sig_node['_toc_parts']
+ if config.toc_object_entries_show_parents == 'domain':
+ return sig_node.get('fullname', name) + parens
+ if config.toc_object_entries_show_parents == 'hide':
+ return name + parens
+ if config.toc_object_entries_show_parents == 'all':
+ return '.'.join([*parents, name + parens])
+ return ''
diff --git a/sphinx/domains/rst.py b/sphinx/domains/rst.py
index fb22cbab5..99d995d48 100644
--- a/sphinx/domains/rst.py
+++ b/sphinx/domains/rst.py
@@ -1,8 +1,12 @@
"""The reStructuredText domain."""
+
from __future__ import annotations
+
import re
from typing import TYPE_CHECKING, Any, ClassVar, cast
+
from docutils.parsers.rst import directives
+
from sphinx import addnodes
from sphinx.directives import ObjectDescription
from sphinx.domains import Domain, ObjType
@@ -10,35 +14,99 @@ from sphinx.locale import _, __
from sphinx.roles import XRefRole
from sphinx.util import logging
from sphinx.util.nodes import make_id, make_refnode
+
if TYPE_CHECKING:
from collections.abc import Iterator
+
from docutils.nodes import Element
+
from sphinx.addnodes import desc_signature, pending_xref
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata, OptionSpec
+
logger = logging.getLogger(__name__)
-dir_sig_re = re.compile('\\.\\. (.+?)::(.*)$')
+
+dir_sig_re = re.compile(r'\.\. (.+?)::(.*)$')
class ReSTMarkup(ObjectDescription[str]):
"""
Description of generic reST markup.
"""
- option_spec: ClassVar[OptionSpec] = {'no-index': directives.flag,
- 'no-index-entry': directives.flag, 'no-contents-entry': directives.
- flag, 'no-typesetting': directives.flag, 'noindex': directives.flag,
- 'noindexentry': directives.flag, 'nocontentsentry': directives.flag}
+ option_spec: ClassVar[OptionSpec] = {
+ 'no-index': directives.flag,
+ 'no-index-entry': directives.flag,
+ 'no-contents-entry': directives.flag,
+ 'no-typesetting': directives.flag,
+ 'noindex': directives.flag,
+ 'noindexentry': directives.flag,
+ 'nocontentsentry': directives.flag,
+ }
+
+ def add_target_and_index(self, name: str, sig: str, signode: desc_signature) -> None:
+ node_id = make_id(self.env, self.state.document, self.objtype, name)
+ signode['ids'].append(node_id)
+ self.state.document.note_explicit_target(signode)
+
+ domain = cast(ReSTDomain, self.env.get_domain('rst'))
+ domain.note_object(self.objtype, name, node_id, location=signode)
-def parse_directive(d: str) ->tuple[str, str]:
+ if 'no-index-entry' not in self.options:
+ indextext = self.get_index_text(self.objtype, name)
+ if indextext:
+ self.indexnode['entries'].append(('single', indextext, node_id, '', None))
+
+ def get_index_text(self, objectname: str, name: str) -> str:
+ return ''
+
+ def _object_hierarchy_parts(self, sig_node: desc_signature) -> tuple[str, ...]:
+ if 'fullname' not in sig_node:
+ return ()
+ directive_names = []
+ for parent in self.env.ref_context.get('rst:directives', ()):
+ directive_names += parent.split(':')
+ name = sig_node['fullname']
+ return tuple(directive_names + name.split(':'))
+
+ def _toc_entry_name(self, sig_node: desc_signature) -> str:
+ if not sig_node.get('_toc_parts'):
+ return ''
+
+ config = self.env.app.config
+ objtype = sig_node.parent.get('objtype')
+ *parents, name = sig_node['_toc_parts']
+ if objtype == 'directive:option':
+ return f':{name}:'
+ if config.toc_object_entries_show_parents in {'domain', 'all'}:
+ name = ':'.join(sig_node['_toc_parts'])
+ if objtype == 'role':
+ return f':{name}:'
+ if objtype == 'directive':
+ return f'.. {name}::'
+ return ''
+
+
+def parse_directive(d: str) -> tuple[str, str]:
"""Parse a directive signature.
Returns (directive, arguments) string tuple. If no arguments are given,
returns (directive, '').
"""
- pass
+ dir = d.strip()
+ if not dir.startswith('.'):
+ # Assume it is a directive without syntax
+ return (dir, '')
+ m = dir_sig_re.match(dir)
+ if not m:
+ return (dir, '')
+ parsed_dir, parsed_args = m.groups()
+ if parsed_args.strip():
+ return (parsed_dir.strip(), ' ' + parsed_args.strip())
+ else:
+ return (parsed_dir.strip(), '')
class ReSTDirective(ReSTMarkup):
@@ -46,13 +114,88 @@ class ReSTDirective(ReSTMarkup):
Description of a reST directive.
"""
+ def handle_signature(self, sig: str, signode: desc_signature) -> str:
+ name, args = parse_directive(sig)
+ desc_name = f'.. {name}::'
+ signode['fullname'] = name.strip()
+ signode += addnodes.desc_name(desc_name, desc_name)
+ if len(args) > 0:
+ signode += addnodes.desc_addname(args, args)
+ return name
+
+ def get_index_text(self, objectname: str, name: str) -> str:
+ return _('%s (directive)') % name
+
+ def before_content(self) -> None:
+ if self.names:
+ directives = self.env.ref_context.setdefault('rst:directives', [])
+ directives.append(self.names[0])
+
+ def after_content(self) -> None:
+ directives = self.env.ref_context.setdefault('rst:directives', [])
+ if directives:
+ directives.pop()
+
class ReSTDirectiveOption(ReSTMarkup):
"""
Description of an option for reST directive.
"""
+
option_spec: ClassVar[OptionSpec] = ReSTMarkup.option_spec.copy()
- option_spec.update({'type': directives.unchanged})
+ option_spec.update({
+ 'type': directives.unchanged,
+ })
+
+ def handle_signature(self, sig: str, signode: desc_signature) -> str:
+ try:
+ name, argument = re.split(r'\s*:\s+', sig.strip(), maxsplit=1)
+ except ValueError:
+ name, argument = sig, None
+
+ desc_name = f':{name}:'
+ signode['fullname'] = name.strip()
+ signode += addnodes.desc_name(desc_name, desc_name)
+ if argument:
+ signode += addnodes.desc_annotation(' ' + argument, ' ' + argument)
+ if self.options.get('type'):
+ text = ' (%s)' % self.options['type']
+ signode += addnodes.desc_annotation(text, text)
+ return name
+
+ def add_target_and_index(self, name: str, sig: str, signode: desc_signature) -> None:
+ domain = cast(ReSTDomain, self.env.get_domain('rst'))
+
+ directive_name = self.current_directive
+ if directive_name:
+ prefix = f'{self.objtype}-{directive_name}'
+ objname = f'{directive_name}:{name}'
+ else:
+ prefix = self.objtype
+ objname = name
+
+ node_id = make_id(self.env, self.state.document, prefix, name)
+ signode['ids'].append(node_id)
+ self.state.document.note_explicit_target(signode)
+ domain.note_object(self.objtype, objname, node_id, location=signode)
+
+ if directive_name:
+ key = name[0].upper()
+ pair = [_('%s (directive)') % directive_name,
+ _(':%s: (directive option)') % name]
+ self.indexnode['entries'].append(('pair', '; '.join(pair), node_id, '', key))
+ else:
+ key = name[0].upper()
+ text = _(':%s: (directive option)') % name
+ self.indexnode['entries'].append(('single', text, node_id, '', key))
+
+ @property
+ def current_directive(self) -> str:
+ directives = self.env.ref_context.get('rst:directives')
+ if directives:
+ return directives[-1]
+ else:
+ return ''
class ReSTRole(ReSTMarkup):
@@ -60,15 +203,102 @@ class ReSTRole(ReSTMarkup):
Description of a reST role.
"""
+ def handle_signature(self, sig: str, signode: desc_signature) -> str:
+ desc_name = f':{sig}:'
+ signode['fullname'] = sig.strip()
+ signode += addnodes.desc_name(desc_name, desc_name)
+ return sig
+
+ def get_index_text(self, objectname: str, name: str) -> str:
+ return _('%s (role)') % name
+
class ReSTDomain(Domain):
"""ReStructuredText domain."""
+
name = 'rst'
label = 'reStructuredText'
- object_types = {'directive': ObjType(_('directive'), 'dir'),
- 'directive:option': ObjType(_('directive-option'), 'dir'), 'role':
- ObjType(_('role'), 'role')}
- directives = {'directive': ReSTDirective, 'directive:option':
- ReSTDirectiveOption, 'role': ReSTRole}
- roles = {'dir': XRefRole(), 'role': XRefRole()}
- initial_data: dict[str, dict[tuple[str, str], str]] = {'objects': {}}
+
+ object_types = {
+ 'directive': ObjType(_('directive'), 'dir'),
+ 'directive:option': ObjType(_('directive-option'), 'dir'),
+ 'role': ObjType(_('role'), 'role'),
+ }
+ directives = {
+ 'directive': ReSTDirective,
+ 'directive:option': ReSTDirectiveOption,
+ 'role': ReSTRole,
+ }
+ roles = {
+ 'dir': XRefRole(),
+ 'role': XRefRole(),
+ }
+ initial_data: dict[str, dict[tuple[str, str], str]] = {
+ 'objects': {}, # fullname -> docname, objtype
+ }
+
+ @property
+ def objects(self) -> dict[tuple[str, str], tuple[str, str]]:
+ return self.data.setdefault('objects', {}) # (objtype, fullname) -> (docname, node_id)
+
+ def note_object(self, objtype: str, name: str, node_id: str, location: Any = None) -> None:
+ if (objtype, name) in self.objects:
+ docname, node_id = self.objects[objtype, name]
+ logger.warning(__('duplicate description of %s %s, other instance in %s'),
+ objtype, name, docname, location=location)
+
+ self.objects[objtype, name] = (self.env.docname, node_id)
+
+ def clear_doc(self, docname: str) -> None:
+ for (typ, name), (doc, _node_id) in list(self.objects.items()):
+ if doc == docname:
+ del self.objects[typ, name]
+
+ def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> None:
+ # XXX check duplicates
+ for (typ, name), (doc, node_id) in otherdata['objects'].items():
+ if doc in docnames:
+ self.objects[typ, name] = (doc, node_id)
+
+ def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
+ typ: str, target: str, node: pending_xref, contnode: Element,
+ ) -> Element | None:
+ objtypes = self.objtypes_for_role(typ)
+ if not objtypes:
+ return None
+ for objtype in objtypes:
+ result = self.objects.get((objtype, target))
+ if result:
+ todocname, node_id = result
+ return make_refnode(builder, fromdocname, todocname, node_id,
+ contnode, target + ' ' + objtype)
+ return None
+
+ def resolve_any_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
+ target: str, node: pending_xref, contnode: Element,
+ ) -> list[tuple[str, Element]]:
+ results: list[tuple[str, Element]] = []
+ for objtype in self.object_types:
+ result = self.objects.get((objtype, target))
+ if result:
+ todocname, node_id = result
+ results.append(
+ ('rst:' + self.role_for_objtype(objtype), # type: ignore[operator]
+ make_refnode(builder, fromdocname, todocname, node_id,
+ contnode, target + ' ' + objtype)))
+ return results
+
+ def get_objects(self) -> Iterator[tuple[str, str, str, str, str, int]]:
+ for (typ, name), (docname, node_id) in self.data['objects'].items():
+ yield name, name, typ, docname, node_id, 1
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_domain(ReSTDomain)
+
+ return {
+ 'version': 'builtin',
+ 'env_version': 2,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/environment/adapters/asset.py b/sphinx/environment/adapters/asset.py
index 524c29ad4..dc0cf7669 100644
--- a/sphinx/environment/adapters/asset.py
+++ b/sphinx/environment/adapters/asset.py
@@ -1,13 +1,16 @@
"""Assets adapter for sphinx.environment."""
+
from sphinx.environment import BuildEnvironment
from sphinx.util._pathlib import _StrPath
class ImageAdapter:
-
- def __init__(self, env: BuildEnvironment) ->None:
+ def __init__(self, env: BuildEnvironment) -> None:
self.env = env
- def get_original_image_uri(self, name: str) ->str:
+ def get_original_image_uri(self, name: str) -> str:
"""Get the original image URI."""
- pass
+ while _StrPath(name) in self.env.original_image_uri:
+ name = self.env.original_image_uri[_StrPath(name)]
+
+ return name
diff --git a/sphinx/environment/adapters/indexentries.py b/sphinx/environment/adapters/indexentries.py
index bb2768a33..64788f51a 100644
--- a/sphinx/environment/adapters/indexentries.py
+++ b/sphinx/environment/adapters/indexentries.py
@@ -1,58 +1,232 @@
"""Index entries adapters for sphinx.environment."""
+
from __future__ import annotations
+
import re
import unicodedata
from itertools import groupby
from typing import TYPE_CHECKING
+
from sphinx.errors import NoUri
from sphinx.locale import _, __
from sphinx.util import logging
from sphinx.util.index_entries import _split_into
+
if TYPE_CHECKING:
from typing import Literal, TypeAlias
+
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
+
_IndexEntryTarget: TypeAlias = tuple[str | None, str | Literal[False]]
_IndexEntryTargets: TypeAlias = list[_IndexEntryTarget]
_IndexEntryCategoryKey: TypeAlias = str | None
- _IndexEntrySubItems: TypeAlias = dict[str, tuple[_IndexEntryTargets,
- _IndexEntryCategoryKey]]
- _IndexEntry: TypeAlias = tuple[_IndexEntryTargets, _IndexEntrySubItems,
- _IndexEntryCategoryKey]
+ _IndexEntrySubItems: TypeAlias = dict[
+ str,
+ tuple[_IndexEntryTargets, _IndexEntryCategoryKey],
+ ]
+ _IndexEntry: TypeAlias = tuple[
+ _IndexEntryTargets,
+ _IndexEntrySubItems,
+ _IndexEntryCategoryKey,
+ ]
_IndexEntryMap: TypeAlias = dict[str, _IndexEntry]
- _Index: TypeAlias = list[tuple[str, list[tuple[str, tuple[
- _IndexEntryTargets, list[tuple[str, _IndexEntryTargets]],
- _IndexEntryCategoryKey]]]]]
+ _Index: TypeAlias = list[
+ tuple[
+ str,
+ list[
+ tuple[
+ str,
+ tuple[
+ _IndexEntryTargets,
+ list[tuple[str, _IndexEntryTargets]],
+ _IndexEntryCategoryKey
+ ]
+ ]
+ ]
+ ]
+ ]
+
logger = logging.getLogger(__name__)
class IndexEntries:
-
- def __init__(self, env: BuildEnvironment) ->None:
+ def __init__(self, env: BuildEnvironment) -> None:
self.env = env
self.builder: Builder
- def create_index(self, builder: Builder, group_entries: bool=True,
- _fixre: re.Pattern[str]=re.compile('(.*) ([(][^()]*[)])')) ->_Index:
+ def create_index(
+ self,
+ builder: Builder,
+ group_entries: bool = True,
+ _fixre: re.Pattern[str] = re.compile(r'(.*) ([(][^()]*[)])'),
+ ) -> _Index:
"""Create the real index from the collected index entries."""
- pass
+ new: _IndexEntryMap = {}
+
+ rel_uri: str | Literal[False]
+ index_domain = self.env.domains['index']
+ for docname, entries in index_domain.entries.items():
+ try:
+ rel_uri = builder.get_relative_uri('genindex', docname)
+ except NoUri:
+ rel_uri = False
+
+ # new entry types must be listed in directives/other.py!
+ for entry_type, value, target_id, main, category_key in entries:
+ uri = rel_uri is not False and f'{rel_uri}#{target_id}'
+ try:
+ if entry_type == 'single':
+ try:
+ entry, sub_entry = _split_into(2, 'single', value)
+ except ValueError:
+ entry, = _split_into(1, 'single', value)
+ sub_entry = ''
+ _add_entry(entry, sub_entry, main,
+ dic=new, link=uri, key=category_key)
+ elif entry_type == 'pair':
+ first, second = _split_into(2, 'pair', value)
+ _add_entry(first, second, main,
+ dic=new, link=uri, key=category_key)
+ _add_entry(second, first, main,
+ dic=new, link=uri, key=category_key)
+ elif entry_type == 'triple':
+ first, second, third = _split_into(3, 'triple', value)
+ _add_entry(first, second + ' ' + third, main,
+ dic=new, link=uri, key=category_key)
+ _add_entry(second, third + ', ' + first, main,
+ dic=new, link=uri, key=category_key)
+ _add_entry(third, first + ' ' + second, main,
+ dic=new, link=uri, key=category_key)
+ elif entry_type == 'see':
+ first, second = _split_into(2, 'see', value)
+ _add_entry(first, _('see %s') % second, None,
+ dic=new, link=False, key=category_key)
+ elif entry_type == 'seealso':
+ first, second = _split_into(2, 'see', value)
+ _add_entry(first, _('see also %s') % second, None,
+ dic=new, link=False, key=category_key)
+ else:
+ logger.warning(__('unknown index entry type %r'), entry_type,
+ location=docname)
+ except ValueError as err:
+ logger.warning(str(err), location=docname)
+ for (targets, sub_items, _category_key) in new.values():
+ targets.sort(key=_key_func_0)
+ for (sub_targets, _sub_category_key) in sub_items.values():
+ sub_targets.sort(key=_key_func_0)
-def _key_func_0(entry: _IndexEntryTarget) ->tuple[bool, str | Literal[False]]:
+ new_list: list[tuple[str, _IndexEntry]] = sorted(new.items(), key=_key_func_1)
+
+ if group_entries:
+ # fixup entries: transform
+ # func() (in module foo)
+ # func() (in module bar)
+ # into
+ # func()
+ # (in module foo)
+ # (in module bar)
+ old_key = ''
+ old_sub_items: _IndexEntrySubItems = {}
+ i = 0
+ while i < len(new_list):
+ key, (targets, sub_items, category_key) = new_list[i]
+ # cannot move if it has sub_items; structure gets too complex
+ if not sub_items:
+ m = _fixre.match(key)
+ if m:
+ if old_key == m.group(1):
+ # prefixes match: add entry as subitem of the
+ # previous entry
+ old_sub_items.setdefault(
+ m.group(2), ([], category_key))[0].extend(targets)
+ del new_list[i]
+ continue
+ old_key = m.group(1)
+ else:
+ old_key = key
+ old_sub_items = sub_items
+ i += 1
+
+ grouped = []
+ for (group_key, group) in groupby(new_list, _group_by_func):
+ group_list = []
+ for group_entry in group:
+ entry_key, (targets, sub_items, category_key) = group_entry
+ pairs = [
+ (sub_key, sub_targets)
+ for (sub_key, (sub_targets, _sub_category_key))
+ in sub_items.items()
+ ]
+ pairs.sort(key=_key_func_2)
+ group_list.append((entry_key, (targets, pairs, category_key)))
+ grouped.append((group_key, group_list))
+ return grouped
+
+
+def _add_entry(word: str, subword: str, main: str | None, *,
+ dic: _IndexEntryMap,
+ link: str | Literal[False], key: _IndexEntryCategoryKey) -> None:
+ entry = dic.setdefault(word, ([], {}, key))
+ if subword:
+ targets = entry[1].setdefault(subword, ([], key))[0]
+ else:
+ targets = entry[0]
+ if link:
+ targets.append((main, link))
+
+
+def _key_func_0(entry: _IndexEntryTarget) -> tuple[bool, str | Literal[False]]:
"""Sort the index entries for same keyword."""
- pass
+ main, uri = entry
+ return not main, uri # show main entries at first
-def _key_func_1(entry: tuple[str, _IndexEntry]) ->tuple[tuple[int, str], str]:
+def _key_func_1(entry: tuple[str, _IndexEntry]) -> tuple[tuple[int, str], str]:
"""Sort the index entries"""
- pass
+ key, (_targets, _sub_items, category_key) = entry
+ if category_key:
+ # using the specified category key to sort
+ key = category_key
+ lc_key = unicodedata.normalize('NFD', key.lower())
+ if lc_key.startswith('\N{RIGHT-TO-LEFT MARK}'):
+ lc_key = lc_key[1:]
+
+ if not lc_key[0:1].isalpha() and not lc_key.startswith('_'):
+ # put symbols at the front of the index (0)
+ group = 0
+ else:
+ # put non-symbol characters at the following group (1)
+ group = 1
+ # ensure a deterministic order *within* letters by also sorting on
+ # the entry itself
+ return (group, lc_key), entry[0]
-def _key_func_2(entry: tuple[str, _IndexEntryTargets]) ->str:
+def _key_func_2(entry: tuple[str, _IndexEntryTargets]) -> str:
"""Sort the sub-index entries"""
- pass
+ key = unicodedata.normalize('NFD', entry[0].lower())
+ if key.startswith('\N{RIGHT-TO-LEFT MARK}'):
+ key = key[1:]
+ if key[0:1].isalpha() or key.startswith('_'):
+ key = chr(127) + key
+ return key
-def _group_by_func(entry: tuple[str, _IndexEntry]) ->str:
+def _group_by_func(entry: tuple[str, _IndexEntry]) -> str:
"""Group the entries by letter or category key."""
- pass
+ key, (targets, sub_items, category_key) = entry
+
+ if category_key is not None:
+ return category_key
+
+ # now calculate the key
+ if key.startswith('\N{RIGHT-TO-LEFT MARK}'):
+ key = key[1:]
+ letter = unicodedata.normalize('NFD', key[0])[0].upper()
+ if letter.isalpha() or letter == '_':
+ return letter
+
+ # get all other symbols under one heading
+ return _('Symbols')
diff --git a/sphinx/environment/adapters/toctree.py b/sphinx/environment/adapters/toctree.py
index e3afbc450..217cb0d41 100644
--- a/sphinx/environment/adapters/toctree.py
+++ b/sphinx/environment/adapters/toctree.py
@@ -1,53 +1,109 @@
"""Toctree adapter for sphinx.environment."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any, TypeVar
+
from docutils import nodes
from docutils.nodes import Element, Node
+
from sphinx import addnodes
from sphinx.locale import __
from sphinx.util import logging, url_re
from sphinx.util.matching import Matcher
from sphinx.util.nodes import _only_node_keep_children, clean_astext
+
if TYPE_CHECKING:
from collections.abc import Iterable, Set
+
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
from sphinx.util.tags import Tags
+
+
logger = logging.getLogger(__name__)
-def note_toctree(env: BuildEnvironment, docname: str, toctreenode: addnodes
- .toctree) ->None:
+def note_toctree(env: BuildEnvironment, docname: str, toctreenode: addnodes.toctree) -> None:
"""Note a TOC tree directive in a document and gather information about
file relations from it.
"""
- pass
+ if toctreenode['glob']:
+ env.glob_toctrees.add(docname)
+ if toctreenode.get('numbered'):
+ env.numbered_toctrees.add(docname)
+ include_files = toctreenode['includefiles']
+ for include_file in include_files:
+ # note that if the included file is rebuilt, this one must be
+ # too (since the TOC of the included file could have changed)
+ env.files_to_rebuild.setdefault(include_file, set()).add(docname)
+ env.toctree_includes.setdefault(docname, []).extend(include_files)
-def document_toc(env: BuildEnvironment, docname: str, tags: Tags) ->Node:
+def document_toc(env: BuildEnvironment, docname: str, tags: Tags) -> Node:
"""Get the (local) table of contents for a document.
Note that this is only the sections within the document.
For a ToC tree that shows the document's place in the
ToC structure, use `get_toctree_for`.
"""
- pass
+ tocdepth = env.metadata[docname].get('tocdepth', 0)
+ try:
+ toc = _toctree_copy(env.tocs[docname], 2, tocdepth, False, tags)
+ except KeyError:
+ # the document does not exist any more:
+ # return a dummy node that renders to nothing
+ return nodes.paragraph()
+
+ for node in toc.findall(nodes.reference):
+ node['refuri'] = node['anchorname'] or '#'
+ return toc
-def global_toctree_for_doc(env: BuildEnvironment, docname: str, builder:
- Builder, collapse: bool=False, includehidden: bool=True, maxdepth: int=
- 0, titles_only: bool=False) ->(Element | None):
+def global_toctree_for_doc(
+ env: BuildEnvironment,
+ docname: str,
+ builder: Builder,
+ collapse: bool = False,
+ includehidden: bool = True,
+ maxdepth: int = 0,
+ titles_only: bool = False,
+) -> Element | None:
"""Get the global ToC tree at a given document.
This gives the global ToC, with all ancestors and their siblings.
"""
- pass
+ resolved = (
+ _resolve_toctree(
+ env,
+ docname,
+ builder,
+ toctree_node,
+ prune=True,
+ maxdepth=int(maxdepth),
+ titles_only=titles_only,
+ collapse=collapse,
+ includehidden=includehidden,
+ )
+ for toctree_node in env.master_doctree.findall(addnodes.toctree)
+ )
+ toctrees = [
+ toctree for toctree in resolved if toctree is not None
+ ]
+ if not toctrees:
+ return None
+ result = toctrees[0]
+ for toctree in toctrees[1:]:
+ result.extend(toctree.children)
+ return result
-def _resolve_toctree(env: BuildEnvironment, docname: str, builder: Builder,
- toctree: addnodes.toctree, *, prune: bool=True, maxdepth: int=0,
- titles_only: bool=False, collapse: bool=False, includehidden: bool=False
- ) ->(Element | None):
+
+def _resolve_toctree(
+ env: BuildEnvironment, docname: str, builder: Builder, toctree: addnodes.toctree, *,
+ prune: bool = True, maxdepth: int = 0, titles_only: bool = False,
+ collapse: bool = False, includehidden: bool = False,
+) -> Element | None:
"""Resolve a *toctree* node into individual bullet lists with titles
as items, returning None (if no containing titles are found) or
a new node.
@@ -59,33 +115,407 @@ def _resolve_toctree(env: BuildEnvironment, docname: str, builder: Builder,
If *collapse* is True, all branches not containing docname will
be collapsed.
"""
- pass
+ if toctree.get('hidden', False) and not includehidden:
+ return None
+
+ # For reading the following two helper function, it is useful to keep
+ # in mind the node structure of a toctree (using HTML-like node names
+ # for brevity):
+ #
+ # <ul>
+ # <li>
+ # <p><a></p>
+ # <p><a></p>
+ # ...
+ # <ul>
+ # ...
+ # </ul>
+ # </li>
+ # </ul>
+ #
+ # The transformation is made in two passes in order to avoid
+ # interactions between marking and pruning the tree (see bug #1046).
+
+ toctree_ancestors = _get_toctree_ancestors(env.toctree_includes, docname)
+ included = Matcher(env.config.include_patterns)
+ excluded = Matcher(env.config.exclude_patterns)
+ maxdepth = maxdepth or toctree.get('maxdepth', -1)
+ if not titles_only and toctree.get('titlesonly', False):
+ titles_only = True
+ if not includehidden and toctree.get('includehidden', False):
+ includehidden = True
-def _entries_from_toctree(env: BuildEnvironment, prune: bool, titles_only:
- bool, collapse: bool, includehidden: bool, tags: Tags,
- toctree_ancestors: Set[str], included: Matcher, excluded: Matcher,
- toctreenode: addnodes.toctree, parents: list[str], subtree: bool=False
- ) ->list[Element]:
+ tocentries = _entries_from_toctree(
+ env,
+ prune,
+ titles_only,
+ collapse,
+ includehidden,
+ builder.tags,
+ toctree_ancestors,
+ included,
+ excluded,
+ toctree,
+ [],
+ )
+ if not tocentries:
+ return None
+
+ newnode = addnodes.compact_paragraph('', '')
+ if caption := toctree.attributes.get('caption'):
+ caption_node = nodes.title(caption, '', *[nodes.Text(caption)])
+ caption_node.line = toctree.line
+ caption_node.source = toctree.source
+ caption_node.rawsource = toctree['rawcaption']
+ if hasattr(toctree, 'uid'):
+ # move uid to caption_node to translate it
+ caption_node.uid = toctree.uid # type: ignore[attr-defined]
+ del toctree.uid
+ newnode.append(caption_node)
+ newnode.extend(tocentries)
+ newnode['toctree'] = True
+
+ # prune the tree to maxdepth, also set toc depth and current classes
+ _toctree_add_classes(newnode, 1, docname)
+ newnode = _toctree_copy(newnode, 1, maxdepth if prune else 0, collapse, builder.tags)
+
+ if isinstance(newnode[-1], nodes.Element) and len(newnode[-1]) == 0: # No titles found
+ return None
+
+ # set the target paths in the toctrees (they are not known at TOC
+ # generation time)
+ for refnode in newnode.findall(nodes.reference):
+ if url_re.match(refnode['refuri']) is None:
+ rel_uri = builder.get_relative_uri(docname, refnode['refuri'])
+ refnode['refuri'] = rel_uri + refnode['anchorname']
+ return newnode
+
+
+def _entries_from_toctree(
+ env: BuildEnvironment,
+ prune: bool,
+ titles_only: bool,
+ collapse: bool,
+ includehidden: bool,
+ tags: Tags,
+ toctree_ancestors: Set[str],
+ included: Matcher,
+ excluded: Matcher,
+ toctreenode: addnodes.toctree,
+ parents: list[str],
+ subtree: bool = False,
+) -> list[Element]:
"""Return TOC entries for a toctree node."""
- pass
+ entries: list[Element] = []
+ for (title, ref) in toctreenode['entries']:
+ try:
+ toc, refdoc = _toctree_entry(
+ title, ref, env, prune, collapse, tags, toctree_ancestors,
+ included, excluded, toctreenode, parents,
+ )
+ except LookupError:
+ continue
+
+ # children of toc are:
+ # - list_item + compact_paragraph + (reference and subtoc)
+ # - only + subtoc
+ # - toctree
+ children: Iterable[nodes.Element] = toc.children # type: ignore[assignment]
+
+ # if titles_only is given, only keep the main title and
+ # sub-toctrees
+ if titles_only:
+ # delete everything but the toplevel title(s)
+ # and toctrees
+ for top_level in children:
+ # nodes with length 1 don't have any children anyway
+ if len(top_level) > 1:
+ if subtrees := list(top_level.findall(addnodes.toctree)):
+ top_level[1][:] = subtrees # type: ignore[index]
+ else:
+ top_level.pop(1)
+ # resolve all sub-toctrees
+ for sub_toc_node in list(toc.findall(addnodes.toctree)):
+ if sub_toc_node.get('hidden', False) and not includehidden:
+ continue
+ for i, entry in enumerate(
+ _entries_from_toctree(
+ env,
+ prune,
+ titles_only,
+ collapse,
+ includehidden,
+ tags,
+ toctree_ancestors,
+ included,
+ excluded,
+ sub_toc_node,
+ [refdoc, *parents],
+ subtree=True,
+ ),
+ start=sub_toc_node.parent.index(sub_toc_node) + 1,
+ ):
+ sub_toc_node.parent.insert(i, entry)
+ sub_toc_node.parent.remove(sub_toc_node)
+
+ entries.extend(children)
+
+ if not subtree:
+ ret = nodes.bullet_list()
+ ret += entries
+ return [ret]
+
+ return entries
+
+def _toctree_entry(
+ title: str,
+ ref: str,
+ env: BuildEnvironment,
+ prune: bool,
+ collapse: bool,
+ tags: Tags,
+ toctree_ancestors: Set[str],
+ included: Matcher,
+ excluded: Matcher,
+ toctreenode: addnodes.toctree,
+ parents: list[str],
+) -> tuple[Element, str]:
+ from sphinx.domains.std import StandardDomain
-def _toctree_add_classes(node: Element, depth: int, docname: str) ->None:
+ try:
+ refdoc = ''
+ if url_re.match(ref):
+ toc = _toctree_url_entry(title, ref)
+ elif ref == 'self':
+ toc = _toctree_self_entry(title, toctreenode['parent'], env.titles)
+ elif ref in StandardDomain._virtual_doc_names:
+ toc = _toctree_generated_entry(title, ref)
+ else:
+ if ref in parents:
+ logger.warning(__('circular toctree references '
+ 'detected, ignoring: %s <- %s'),
+ ref, ' <- '.join(parents),
+ location=ref, type='toc', subtype='circular')
+ msg = 'circular reference'
+ raise LookupError(msg)
+
+ toc, refdoc = _toctree_standard_entry(
+ title,
+ ref,
+ env.metadata[ref].get('tocdepth', 0),
+ env.tocs[ref],
+ toctree_ancestors,
+ prune,
+ collapse,
+ tags,
+ )
+
+ if not toc.children:
+ # empty toc means: no titles will show up in the toctree
+ logger.warning(__('toctree contains reference to document %r that '
+ "doesn't have a title: no link will be generated"),
+ ref, location=toctreenode, type='toc', subtype='no_title')
+ except KeyError:
+ # this is raised if the included file does not exist
+ ref_path = str(env.doc2path(ref, False))
+ if excluded(ref_path):
+ message = __('toctree contains reference to excluded document %r')
+ elif not included(ref_path):
+ message = __('toctree contains reference to non-included document %r')
+ else:
+ message = __('toctree contains reference to nonexisting document %r')
+
+ logger.warning(message, ref, location=toctreenode)
+ raise
+ return toc, refdoc
+
+
+def _toctree_url_entry(title: str, ref: str) -> nodes.bullet_list:
+ if title is None:
+ title = ref
+ reference = nodes.reference('', '', internal=False,
+ refuri=ref, anchorname='',
+ *[nodes.Text(title)])
+ para = addnodes.compact_paragraph('', '', reference)
+ item = nodes.list_item('', para)
+ toc = nodes.bullet_list('', item)
+ return toc
+
+
+def _toctree_self_entry(
+ title: str, ref: str, titles: dict[str, nodes.title],
+) -> nodes.bullet_list:
+ # 'self' refers to the document from which this
+ # toctree originates
+ if not title:
+ title = clean_astext(titles[ref])
+ reference = nodes.reference('', '', internal=True,
+ refuri=ref,
+ anchorname='',
+ *[nodes.Text(title)])
+ para = addnodes.compact_paragraph('', '', reference)
+ item = nodes.list_item('', para)
+ # don't show subitems
+ toc = nodes.bullet_list('', item)
+ return toc
+
+
+def _toctree_generated_entry(title: str, ref: str) -> nodes.bullet_list:
+ from sphinx.domains.std import StandardDomain
+
+ docname, sectionname = StandardDomain._virtual_doc_names[ref]
+ if not title:
+ title = sectionname
+ reference = nodes.reference('', title, internal=True,
+ refuri=docname, anchorname='')
+ para = addnodes.compact_paragraph('', '', reference)
+ item = nodes.list_item('', para)
+ # don't show subitems
+ toc = nodes.bullet_list('', item)
+ return toc
+
+
+def _toctree_standard_entry(
+ title: str,
+ ref: str,
+ maxdepth: int,
+ toc: nodes.bullet_list,
+ toctree_ancestors: Set[str],
+ prune: bool,
+ collapse: bool,
+ tags: Tags,
+) -> tuple[nodes.bullet_list, str]:
+ refdoc = ref
+ if ref in toctree_ancestors and (not prune or maxdepth <= 0):
+ toc = toc.deepcopy()
+ else:
+ toc = _toctree_copy(toc, 2, maxdepth, collapse, tags)
+
+ if title and toc.children and len(toc.children) == 1:
+ child = toc.children[0]
+ for refnode in child.findall(nodes.reference):
+ if refnode['refuri'] == ref and not refnode['anchorname']:
+ refnode.children[:] = [nodes.Text(title)]
+ return toc, refdoc
+
+
+def _toctree_add_classes(node: Element, depth: int, docname: str) -> None:
"""Add 'toctree-l%d' and 'current' classes to the toctree."""
- pass
+ for subnode in node.children:
+ if isinstance(subnode, addnodes.compact_paragraph | nodes.list_item):
+ # for <p> and <li>, indicate the depth level and recurse
+ subnode['classes'].append(f'toctree-l{depth - 1}')
+ _toctree_add_classes(subnode, depth, docname)
+ elif isinstance(subnode, nodes.bullet_list):
+ # for <ul>, just recurse
+ _toctree_add_classes(subnode, depth + 1, docname)
+ elif isinstance(subnode, nodes.reference):
+ # for <a>, identify which entries point to the current
+ # document and therefore may not be collapsed
+ if subnode['refuri'] == docname:
+ if not subnode['anchorname']:
+ # give the whole branch a 'current' class
+ # (useful for styling it differently)
+ branchnode: Element = subnode
+ while branchnode:
+ branchnode['classes'].append('current')
+ branchnode = branchnode.parent
+ # mark the list_item as "on current page"
+ if subnode.parent.parent.get('iscurrent'):
+ # but only if it's not already done
+ return
+ while subnode:
+ subnode['iscurrent'] = True
+ subnode = subnode.parent
ET = TypeVar('ET', bound=Element)
-def _toctree_copy(node: ET, depth: int, maxdepth: int, collapse: bool, tags:
- Tags) ->ET:
+def _toctree_copy(node: ET, depth: int, maxdepth: int, collapse: bool, tags: Tags) -> ET:
"""Utility: Cut and deep-copy a TOC at a specified depth."""
- pass
+ keep_bullet_list_sub_nodes = (depth <= 1
+ or ((depth <= maxdepth or maxdepth <= 0)
+ and (not collapse or 'iscurrent' in node)))
+ copy = node.copy()
+ for subnode in node.children:
+ if isinstance(subnode, addnodes.compact_paragraph | nodes.list_item):
+ # for <p> and <li>, just recurse
+ copy.append(_toctree_copy(subnode, depth, maxdepth, collapse, tags))
+ elif isinstance(subnode, nodes.bullet_list):
+ # for <ul>, copy if the entry is top-level
+ # or, copy if the depth is within bounds and;
+ # collapsing is disabled or the sub-entry's parent is 'current'.
+ # The boolean is constant so is calculated outwith the loop.
+ if keep_bullet_list_sub_nodes:
+ copy.append(_toctree_copy(subnode, depth + 1, maxdepth, collapse, tags))
+ elif isinstance(subnode, addnodes.toctree):
+ # copy sub toctree nodes for later processing
+ copy.append(subnode.copy())
+ elif isinstance(subnode, addnodes.only):
+ # only keep children if the only node matches the tags
+ if _only_node_keep_children(subnode, tags):
+ for child in subnode.children:
+ copy.append(_toctree_copy(
+ child, depth, maxdepth, collapse, tags, # type: ignore[type-var]
+ ))
+ elif isinstance(subnode, nodes.reference | nodes.title):
+ # deep copy references and captions
+ sub_node_copy = subnode.copy()
+ sub_node_copy.children = [child.deepcopy() for child in subnode.children]
+ for child in sub_node_copy.children:
+ child.parent = sub_node_copy
+ copy.append(sub_node_copy)
+ else:
+ msg = f'Unexpected node type {subnode.__class__.__name__!r}!'
+ raise ValueError(msg)
+ return copy
+
+
+def _get_toctree_ancestors(
+ toctree_includes: dict[str, list[str]], docname: str,
+) -> Set[str]:
+ parent: dict[str, str] = {}
+ for p, children in toctree_includes.items():
+ parent |= dict.fromkeys(children, p)
+ ancestors: list[str] = []
+ d = docname
+ while d in parent and d not in ancestors:
+ ancestors.append(d)
+ d = parent[d]
+ # use dict keys for ordered set operations
+ return dict.fromkeys(ancestors).keys()
-class TocTree:
- def __init__(self, env: BuildEnvironment) ->None:
+class TocTree:
+ def __init__(self, env: BuildEnvironment) -> None:
self.env = env
+
+ def note(self, docname: str, toctreenode: addnodes.toctree) -> None:
+ note_toctree(self.env, docname, toctreenode)
+
+ def resolve(self, docname: str, builder: Builder, toctree: addnodes.toctree,
+ prune: bool = True, maxdepth: int = 0, titles_only: bool = False,
+ collapse: bool = False, includehidden: bool = False) -> Element | None:
+ return _resolve_toctree(
+ self.env, docname, builder, toctree,
+ prune=prune,
+ maxdepth=maxdepth,
+ titles_only=titles_only,
+ collapse=collapse,
+ includehidden=includehidden,
+ )
+
+ def get_toctree_ancestors(self, docname: str) -> list[str]:
+ return [*_get_toctree_ancestors(self.env.toctree_includes, docname)]
+
+ def get_toc_for(self, docname: str, builder: Builder) -> Node:
+ return document_toc(self.env, docname, self.env.app.builder.tags)
+
+ def get_toctree_for(
+ self, docname: str, builder: Builder, collapse: bool, **kwargs: Any,
+ ) -> Element | None:
+ return global_toctree_for_doc(self.env, docname, builder, collapse=collapse, **kwargs)
diff --git a/sphinx/environment/collectors/asset.py b/sphinx/environment/collectors/asset.py
index 7fe3a334b..368e47732 100644
--- a/sphinx/environment/collectors/asset.py
+++ b/sphinx/environment/collectors/asset.py
@@ -1,36 +1,148 @@
"""The image collector for sphinx.environment."""
+
from __future__ import annotations
+
import os
from glob import glob
from os import path
from typing import TYPE_CHECKING
+
from docutils import nodes
from docutils.utils import relative_path
+
from sphinx import addnodes
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.i18n import get_image_filename_for_language, search_image_for_language
from sphinx.util.images import guess_mimetype
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
class ImageCollector(EnvironmentCollector):
"""Image files collector for sphinx.environment."""
- def process_doc(self, app: Sphinx, doctree: nodes.document) ->None:
+ def clear_doc(self, app: Sphinx, env: BuildEnvironment, docname: str) -> None:
+ env.images.purge_doc(docname)
+
+ def merge_other(self, app: Sphinx, env: BuildEnvironment,
+ docnames: set[str], other: BuildEnvironment) -> None:
+ env.images.merge_other(docnames, other.images)
+
+ def process_doc(self, app: Sphinx, doctree: nodes.document) -> None:
"""Process and rewrite image URIs."""
- pass
+ docname = app.env.docname
+
+ for node in doctree.findall(nodes.image):
+ # Map the mimetype to the corresponding image. The writer may
+ # choose the best image from these candidates. The special key * is
+ # set if there is only single candidate to be used by a writer.
+ # The special key ? is set for nonlocal URIs.
+ candidates: dict[str, str] = {}
+ node['candidates'] = candidates
+ imguri = node['uri']
+ if imguri.startswith('data:'):
+ candidates['?'] = imguri
+ continue
+ if imguri.find('://') != -1:
+ candidates['?'] = imguri
+ continue
+
+ if imguri.endswith(os.extsep + '*'):
+ # Update `node['uri']` to a relative path from srcdir
+ # from a relative path from current document.
+ rel_imgpath, full_imgpath = app.env.relfn2path(imguri, docname)
+ node['uri'] = rel_imgpath
+
+ # Search language-specific figures at first
+ i18n_imguri = get_image_filename_for_language(imguri, app.env)
+ _, full_i18n_imgpath = app.env.relfn2path(i18n_imguri, docname)
+ self.collect_candidates(app.env, full_i18n_imgpath, candidates, node)
+
+ self.collect_candidates(app.env, full_imgpath, candidates, node)
+ else:
+ # substitute imguri by figure_language_filename
+ # (ex. foo.png -> foo.en.png)
+ imguri = search_image_for_language(imguri, app.env)
+
+ # Update `node['uri']` to a relative path from srcdir
+ # from a relative path from current document.
+ original_uri = node['uri']
+ node['uri'], _ = app.env.relfn2path(imguri, docname)
+ candidates['*'] = node['uri']
+ if node['uri'] != original_uri:
+ node['original_uri'] = original_uri
+
+ # map image paths to unique image names (so that they can be put
+ # into a single directory)
+ for imgpath in candidates.values():
+ app.env.dependencies[docname].add(imgpath)
+ if not os.access(path.join(app.srcdir, imgpath), os.R_OK):
+ logger.warning(__('image file not readable: %s'), imgpath,
+ location=node, type='image', subtype='not_readable')
+ continue
+ app.env.images.add_file(docname, imgpath)
+
+ def collect_candidates(self, env: BuildEnvironment, imgpath: str,
+ candidates: dict[str, str], node: Node) -> None:
+ globbed: dict[str, list[str]] = {}
+ for filename in glob(imgpath):
+ new_imgpath = relative_path(path.join(env.srcdir, 'dummy'),
+ filename)
+ try:
+ mimetype = guess_mimetype(filename)
+ if mimetype is None:
+ basename, suffix = path.splitext(filename)
+ mimetype = 'image/x-' + suffix[1:]
+ if mimetype not in candidates:
+ globbed.setdefault(mimetype, []).append(new_imgpath)
+ except OSError as err:
+ logger.warning(__('image file %s not readable: %s'), filename, err,
+ location=node, type='image', subtype='not_readable')
+ for key, files in globbed.items():
+ candidates[key] = min(files, key=len) # select by similarity
class DownloadFileCollector(EnvironmentCollector):
"""Download files collector for sphinx.environment."""
- def process_doc(self, app: Sphinx, doctree: nodes.document) ->None:
+ def clear_doc(self, app: Sphinx, env: BuildEnvironment, docname: str) -> None:
+ env.dlfiles.purge_doc(docname)
+
+ def merge_other(self, app: Sphinx, env: BuildEnvironment,
+ docnames: set[str], other: BuildEnvironment) -> None:
+ env.dlfiles.merge_other(docnames, other.dlfiles)
+
+ def process_doc(self, app: Sphinx, doctree: nodes.document) -> None:
"""Process downloadable file paths."""
- pass
+ for node in doctree.findall(addnodes.download_reference):
+ targetname = node['reftarget']
+ if '://' in targetname:
+ node['refuri'] = targetname
+ else:
+ rel_filename, filename = app.env.relfn2path(targetname, app.env.docname)
+ app.env.dependencies[app.env.docname].add(rel_filename)
+ if not os.access(filename, os.R_OK):
+ logger.warning(__('download file not readable: %s'), filename,
+ location=node, type='download', subtype='not_readable')
+ continue
+ node['filename'] = app.env.dlfiles.add_file(app.env.docname, rel_filename)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_env_collector(ImageCollector)
+ app.add_env_collector(DownloadFileCollector)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/environment/collectors/dependencies.py b/sphinx/environment/collectors/dependencies.py
index 25637aa67..33b54b824 100644
--- a/sphinx/environment/collectors/dependencies.py
+++ b/sphinx/environment/collectors/dependencies.py
@@ -1,13 +1,19 @@
"""The dependencies collector components for sphinx.environment."""
+
from __future__ import annotations
+
import os
from os import path
from typing import TYPE_CHECKING
+
from docutils.utils import relative_path
+
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.util.osutil import fs_encoding
+
if TYPE_CHECKING:
from docutils import nodes
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata
@@ -16,6 +22,37 @@ if TYPE_CHECKING:
class DependenciesCollector(EnvironmentCollector):
"""dependencies collector for sphinx.environment."""
- def process_doc(self, app: Sphinx, doctree: nodes.document) ->None:
+ def clear_doc(self, app: Sphinx, env: BuildEnvironment, docname: str) -> None:
+ env.dependencies.pop(docname, None)
+
+ def merge_other(self, app: Sphinx, env: BuildEnvironment,
+ docnames: set[str], other: BuildEnvironment) -> None:
+ for docname in docnames:
+ if docname in other.dependencies:
+ env.dependencies[docname] = other.dependencies[docname]
+
+ def process_doc(self, app: Sphinx, doctree: nodes.document) -> None:
"""Process docutils-generated dependency info."""
- pass
+ cwd = os.getcwd()
+ frompath = path.join(path.normpath(app.srcdir), 'dummy')
+ deps = doctree.settings.record_dependencies
+ if not deps:
+ return
+ for dep in deps.list:
+ # the dependency path is relative to the working dir, so get
+ # one relative to the srcdir
+ if isinstance(dep, bytes):
+ dep = dep.decode(fs_encoding)
+ relpath = relative_path(frompath,
+ path.normpath(path.join(cwd, dep)))
+ app.env.dependencies[app.env.docname].add(relpath)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_env_collector(DependenciesCollector)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/environment/collectors/metadata.py b/sphinx/environment/collectors/metadata.py
index 199b4eb5e..bef35119e 100644
--- a/sphinx/environment/collectors/metadata.py
+++ b/sphinx/environment/collectors/metadata.py
@@ -1,8 +1,13 @@
"""The metadata collector components for sphinx.environment."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, cast
+
from docutils import nodes
+
from sphinx.environment.collectors import EnvironmentCollector
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
@@ -12,9 +17,55 @@ if TYPE_CHECKING:
class MetadataCollector(EnvironmentCollector):
"""metadata collector for sphinx.environment."""
- def process_doc(self, app: Sphinx, doctree: nodes.document) ->None:
+ def clear_doc(self, app: Sphinx, env: BuildEnvironment, docname: str) -> None:
+ env.metadata.pop(docname, None)
+
+ def merge_other(self, app: Sphinx, env: BuildEnvironment,
+ docnames: set[str], other: BuildEnvironment) -> None:
+ for docname in docnames:
+ env.metadata[docname] = other.metadata[docname]
+
+ def process_doc(self, app: Sphinx, doctree: nodes.document) -> None:
"""Process the docinfo part of the doctree as metadata.
Keep processing minimal -- just return what docutils says.
"""
- pass
+ index = doctree.first_child_not_matching_class(nodes.PreBibliographic) # type: ignore[arg-type]
+ if index is None:
+ return
+ elif isinstance(doctree[index], nodes.docinfo):
+ md = app.env.metadata[app.env.docname]
+ for node in doctree[index]: # type: ignore[attr-defined]
+ # nodes are multiply inherited...
+ if isinstance(node, nodes.authors):
+ authors = cast(list[nodes.author], node)
+ md['authors'] = [author.astext() for author in authors]
+ elif isinstance(node, nodes.field):
+ assert len(node) == 2
+ field_name = cast(nodes.field_name, node[0])
+ field_body = cast(nodes.field_body, node[1])
+ md[field_name.astext()] = field_body.astext()
+ elif isinstance(node, nodes.TextElement):
+ # other children must be TextElement
+ # see: https://docutils.sourceforge.io/docs/ref/doctree.html#bibliographic-elements # NoQA: E501
+ md[node.__class__.__name__] = node.astext()
+
+ for name, value in md.items():
+ if name == 'tocdepth':
+ try:
+ value = int(value)
+ except ValueError:
+ value = 0
+ md[name] = value
+
+ doctree.pop(index)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_env_collector(MetadataCollector)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/environment/collectors/title.py b/sphinx/environment/collectors/title.py
index 640a7a3a2..76e3f0379 100644
--- a/sphinx/environment/collectors/title.py
+++ b/sphinx/environment/collectors/title.py
@@ -1,9 +1,14 @@
"""The title collector components for sphinx.environment."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING
+
from docutils import nodes
+
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.transforms import SphinxContentsFilter
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
@@ -13,8 +18,45 @@ if TYPE_CHECKING:
class TitleCollector(EnvironmentCollector):
"""title collector for sphinx.environment."""
- def process_doc(self, app: Sphinx, doctree: nodes.document) ->None:
+ def clear_doc(self, app: Sphinx, env: BuildEnvironment, docname: str) -> None:
+ env.titles.pop(docname, None)
+ env.longtitles.pop(docname, None)
+
+ def merge_other(self, app: Sphinx, env: BuildEnvironment,
+ docnames: set[str], other: BuildEnvironment) -> None:
+ for docname in docnames:
+ env.titles[docname] = other.titles[docname]
+ env.longtitles[docname] = other.longtitles[docname]
+
+ def process_doc(self, app: Sphinx, doctree: nodes.document) -> None:
"""Add a title node to the document (just copy the first section title),
and store that title in the environment.
"""
- pass
+ titlenode = nodes.title()
+ longtitlenode = titlenode
+ # explicit title set with title directive; use this only for
+ # the <title> tag in HTML output
+ if 'title' in doctree:
+ longtitlenode = nodes.title()
+ longtitlenode += nodes.Text(doctree['title'])
+ # look for first section title and use that as the title
+ for node in doctree.findall(nodes.section):
+ visitor = SphinxContentsFilter(doctree)
+ node[0].walkabout(visitor)
+ titlenode += visitor.get_entry_text() # type: ignore[no-untyped-call]
+ break
+ else:
+ # document has no title
+ titlenode += nodes.Text(doctree.get('title', '<no title>'))
+ app.env.titles[app.env.docname] = titlenode
+ app.env.longtitles[app.env.docname] = longtitlenode
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_env_collector(TitleCollector)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/environment/collectors/toctree.py b/sphinx/environment/collectors/toctree.py
index c429eda00..e3f80f251 100644
--- a/sphinx/environment/collectors/toctree.py
+++ b/sphinx/environment/collectors/toctree.py
@@ -1,33 +1,369 @@
"""Toctree collector for sphinx.environment."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, TypeVar, cast
+
from docutils import nodes
+
from sphinx import addnodes
from sphinx.environment.adapters.toctree import note_toctree
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.locale import __
from sphinx.transforms import SphinxContentsFilter
from sphinx.util import logging, url_re
+
if TYPE_CHECKING:
from collections.abc import Sequence
+
from docutils.nodes import Element, Node
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata
+
N = TypeVar('N')
+
logger = logging.getLogger(__name__)
class TocTreeCollector(EnvironmentCollector):
+ def clear_doc(self, app: Sphinx, env: BuildEnvironment, docname: str) -> None:
+ env.tocs.pop(docname, None)
+ env.toc_secnumbers.pop(docname, None)
+ env.toc_fignumbers.pop(docname, None)
+ env.toc_num_entries.pop(docname, None)
+ env.toctree_includes.pop(docname, None)
+ env.glob_toctrees.discard(docname)
+ env.numbered_toctrees.discard(docname)
+
+ for subfn, fnset in list(env.files_to_rebuild.items()):
+ fnset.discard(docname)
+ if not fnset:
+ del env.files_to_rebuild[subfn]
+
+ def merge_other(self, app: Sphinx, env: BuildEnvironment, docnames: set[str],
+ other: BuildEnvironment) -> None:
+ for docname in docnames:
+ env.tocs[docname] = other.tocs[docname]
+ env.toc_num_entries[docname] = other.toc_num_entries[docname]
+ if docname in other.toctree_includes:
+ env.toctree_includes[docname] = other.toctree_includes[docname]
+ if docname in other.glob_toctrees:
+ env.glob_toctrees.add(docname)
+ if docname in other.numbered_toctrees:
+ env.numbered_toctrees.add(docname)
+
+ for subfn, fnset in other.files_to_rebuild.items():
+ env.files_to_rebuild.setdefault(subfn, set()).update(fnset & set(docnames))
- def process_doc(self, app: Sphinx, doctree: nodes.document) ->None:
+ def process_doc(self, app: Sphinx, doctree: nodes.document) -> None:
"""Build a TOC from the doctree and store it in the inventory."""
- pass
+ docname = app.env.docname
+ numentries = [0] # nonlocal again...
- def assign_section_numbers(self, env: BuildEnvironment) ->list[str]:
+ def build_toc(
+ node: Element | Sequence[Element],
+ depth: int = 1,
+ ) -> nodes.bullet_list | None:
+ # list of table of contents entries
+ entries: list[Element] = []
+ for sectionnode in node:
+ # find all toctree nodes in this section and add them
+ # to the toc (just copying the toctree node which is then
+ # resolved in self.get_and_resolve_doctree)
+ if isinstance(sectionnode, nodes.section):
+ title = sectionnode[0]
+ # copy the contents of the section title, but without references
+ # and unnecessary stuff
+ visitor = SphinxContentsFilter(doctree)
+ title.walkabout(visitor)
+ nodetext = visitor.get_entry_text() # type: ignore[no-untyped-call]
+ anchorname = _make_anchor_name(sectionnode['ids'], numentries)
+ # make these nodes:
+ # list_item -> compact_paragraph -> reference
+ reference = nodes.reference(
+ '', '', internal=True, refuri=docname,
+ anchorname=anchorname, *nodetext)
+ para = addnodes.compact_paragraph('', '', reference)
+ item: Element = nodes.list_item('', para)
+ sub_item = build_toc(sectionnode, depth + 1)
+ if sub_item:
+ item += sub_item
+ entries.append(item)
+ # Wrap items under an ``.. only::`` directive in a node for
+ # post-processing
+ elif isinstance(sectionnode, addnodes.only):
+ onlynode = addnodes.only(expr=sectionnode['expr'])
+ blist = build_toc(sectionnode, depth)
+ if blist:
+ onlynode += blist.children
+ entries.append(onlynode)
+ # check within the section for other node types
+ elif isinstance(sectionnode, nodes.Element):
+ # cache of parent node -> list item
+ memo_parents: dict[nodes.Element, nodes.list_item] = {}
+ toctreenode: nodes.Node
+ for toctreenode in sectionnode.findall():
+ if isinstance(toctreenode, nodes.section):
+ continue
+ if isinstance(toctreenode, addnodes.toctree):
+ item = toctreenode.copy()
+ entries.append(item)
+ # important: do the inventory stuff
+ note_toctree(app.env, docname, toctreenode)
+ # add object signatures within a section to the ToC
+ elif isinstance(toctreenode, addnodes.desc):
+ # The desc has one or more nested desc_signature,
+ # and then a desc_content, which again may have desc nodes.
+ # Thus, desc is the one we can bubble up to through parents.
+ entry: nodes.list_item | None = None
+ for sig_node in toctreenode:
+ if not isinstance(sig_node, addnodes.desc_signature):
+ continue
+ # Skip if no name set
+ if not sig_node.get('_toc_name', ''):
+ continue
+ # Skip if explicitly disabled
+ if sig_node.parent.get('no-contents-entry'):
+ continue
+ # Skip entries with no ID (e.g. with :no-index: set)
+ ids = sig_node['ids']
+ if not ids:
+ continue
+
+ anchorname = _make_anchor_name(ids, numentries)
+
+ reference = nodes.reference(
+ '', '', nodes.literal('', sig_node['_toc_name']),
+ internal=True, refuri=docname, anchorname=anchorname)
+ para = addnodes.compact_paragraph('', '', reference,
+ skip_section_number=True)
+ entry = nodes.list_item('', para)
+
+ # Find parent node
+ parent = sig_node.parent
+ while parent not in memo_parents and parent != sectionnode:
+ parent = parent.parent
+ # Note, it may both be the limit and in memo_parents,
+ # prefer memo_parents, so we get the nesting.
+ if parent in memo_parents:
+ root_entry = memo_parents[parent]
+ if isinstance(root_entry[-1], nodes.bullet_list):
+ root_entry[-1].append(entry)
+ else:
+ root_entry.append(nodes.bullet_list('', entry))
+ else:
+ assert parent == sectionnode
+ entries.append(entry)
+
+ # Save the latest desc_signature as the one we put sub entries in.
+ # If there are multiple signatures, then the latest is used.
+ if entry is not None:
+ # are there any desc nodes without desc_signature nodes?
+ memo_parents[toctreenode] = entry
+
+ if entries:
+ return nodes.bullet_list('', *entries)
+ return None
+
+ toc = build_toc(doctree)
+ if toc:
+ app.env.tocs[docname] = toc
+ else:
+ app.env.tocs[docname] = nodes.bullet_list('')
+ app.env.toc_num_entries[docname] = numentries[0]
+
+ def get_updated_docs(self, app: Sphinx, env: BuildEnvironment) -> list[str]:
+ return self.assign_section_numbers(env) + self.assign_figure_numbers(env)
+
+ def assign_section_numbers(self, env: BuildEnvironment) -> list[str]:
"""Assign a section number to each heading under a numbered toctree."""
- pass
+ # a list of all docnames whose section numbers changed
+ rewrite_needed = []
+
+ assigned: set[str] = set()
+ old_secnumbers = env.toc_secnumbers
+ env.toc_secnumbers = {}
+
+ def _walk_toc(
+ node: Element,
+ secnums: dict[str, tuple[int, ...]],
+ depth: int,
+ titlenode: nodes.title | None = None,
+ ) -> None:
+ # titlenode is the title of the document, it will get assigned a
+ # secnumber too, so that it shows up in next/prev/parent rellinks
+ for subnode in node.children:
+ if isinstance(subnode, nodes.bullet_list):
+ numstack.append(0)
+ _walk_toc(subnode, secnums, depth - 1, titlenode)
+ numstack.pop()
+ titlenode = None
+ elif isinstance(subnode, nodes.list_item): # NoQA: SIM114
+ _walk_toc(subnode, secnums, depth, titlenode)
+ titlenode = None
+ elif isinstance(subnode, addnodes.only):
+ # at this stage we don't know yet which sections are going
+ # to be included; just include all of them, even if it leads
+ # to gaps in the numbering
+ _walk_toc(subnode, secnums, depth, titlenode)
+ titlenode = None
+ elif isinstance(subnode, addnodes.compact_paragraph):
+ if 'skip_section_number' in subnode:
+ continue
+ numstack[-1] += 1
+ reference = cast(nodes.reference, subnode[0])
+ if depth > 0:
+ number = numstack.copy()
+ secnums[reference['anchorname']] = tuple(numstack)
+ else:
+ number = None
+ secnums[reference['anchorname']] = ()
+ reference['secnumber'] = number
+ if titlenode:
+ titlenode['secnumber'] = number
+ titlenode = None
+ elif isinstance(subnode, addnodes.toctree):
+ _walk_toctree(subnode, depth)
+
+ def _walk_toctree(toctreenode: addnodes.toctree, depth: int) -> None:
+ if depth == 0:
+ return
+ for (_title, ref) in toctreenode['entries']:
+ if url_re.match(ref) or ref == 'self':
+ # don't mess with those
+ continue
+ if ref in assigned:
+ logger.warning(__('%s is already assigned section numbers '
+ '(nested numbered toctree?)'), ref,
+ location=toctreenode, type='toc', subtype='secnum')
+ elif ref in env.tocs:
+ secnums: dict[str, tuple[int, ...]] = {}
+ env.toc_secnumbers[ref] = secnums
+ assigned.add(ref)
+ _walk_toc(env.tocs[ref], secnums, depth, env.titles.get(ref))
+ if secnums != old_secnumbers.get(ref):
+ rewrite_needed.append(ref)
+
+ for docname in env.numbered_toctrees:
+ assigned.add(docname)
+ doctree = env.get_doctree(docname)
+ for toctreenode in doctree.findall(addnodes.toctree):
+ depth = toctreenode.get('numbered', 0)
+ if depth:
+ # every numbered toctree gets new numbering
+ numstack = [0]
+ _walk_toctree(toctreenode, depth)
- def assign_figure_numbers(self, env: BuildEnvironment) ->list[str]:
+ return rewrite_needed
+
+ def assign_figure_numbers(self, env: BuildEnvironment) -> list[str]:
"""Assign a figure number to each figure under a numbered toctree."""
- pass
+ generated_docnames = frozenset(env.domains['std']._virtual_doc_names)
+
+ rewrite_needed = []
+
+ assigned: set[str] = set()
+ old_fignumbers = env.toc_fignumbers
+ env.toc_fignumbers = {}
+ fignum_counter: dict[str, dict[tuple[int, ...], int]] = {}
+
+ def get_figtype(node: Node) -> str | None:
+ for domain in env.domains.values():
+ figtype = domain.get_enumerable_node_type(node)
+ if (domain.name == 'std'
+ and not domain.get_numfig_title(node)): # type: ignore[attr-defined] # NoQA: E501
+ # Skip if uncaptioned node
+ continue
+
+ if figtype:
+ return figtype
+
+ return None
+
+ def get_section_number(docname: str, section: nodes.section) -> tuple[int, ...]:
+ anchorname = '#' + section['ids'][0]
+ secnumbers = env.toc_secnumbers.get(docname, {})
+ if anchorname in secnumbers:
+ secnum = secnumbers.get(anchorname)
+ else:
+ secnum = secnumbers.get('')
+
+ return secnum or ()
+
+ def get_next_fignumber(figtype: str, secnum: tuple[int, ...]) -> tuple[int, ...]:
+ counter = fignum_counter.setdefault(figtype, {})
+
+ secnum = secnum[:env.config.numfig_secnum_depth]
+ counter[secnum] = counter.get(secnum, 0) + 1
+ return (*secnum, counter[secnum])
+
+ def register_fignumber(docname: str, secnum: tuple[int, ...],
+ figtype: str, fignode: Element) -> None:
+ env.toc_fignumbers.setdefault(docname, {})
+ fignumbers = env.toc_fignumbers[docname].setdefault(figtype, {})
+ figure_id = fignode['ids'][0]
+
+ fignumbers[figure_id] = get_next_fignumber(figtype, secnum)
+
+ def _walk_doctree(docname: str, doctree: Element, secnum: tuple[int, ...]) -> None:
+ nonlocal generated_docnames
+ for subnode in doctree.children:
+ if isinstance(subnode, nodes.section):
+ next_secnum = get_section_number(docname, subnode)
+ if next_secnum:
+ _walk_doctree(docname, subnode, next_secnum)
+ else:
+ _walk_doctree(docname, subnode, secnum)
+ elif isinstance(subnode, addnodes.toctree):
+ for _title, subdocname in subnode['entries']:
+ if url_re.match(subdocname) or subdocname == 'self':
+ # don't mess with those
+ continue
+ if subdocname in generated_docnames:
+ # or these
+ continue
+
+ _walk_doc(subdocname, secnum)
+ elif isinstance(subnode, nodes.Element):
+ figtype = get_figtype(subnode)
+ if figtype and subnode['ids']:
+ register_fignumber(docname, secnum, figtype, subnode)
+
+ _walk_doctree(docname, subnode, secnum)
+
+ def _walk_doc(docname: str, secnum: tuple[int, ...]) -> None:
+ if docname not in assigned:
+ assigned.add(docname)
+ doctree = env.get_doctree(docname)
+ _walk_doctree(docname, doctree, secnum)
+
+ if env.config.numfig:
+ _walk_doc(env.config.root_doc, ())
+ for docname, fignums in env.toc_fignumbers.items():
+ if fignums != old_fignumbers.get(docname):
+ rewrite_needed.append(docname)
+
+ return rewrite_needed
+
+
+def _make_anchor_name(ids: list[str], num_entries: list[int]) -> str:
+ if not num_entries[0]:
+ # for the very first toc entry, don't add an anchor
+ # as it is the file's title anyway
+ anchorname = ''
+ else:
+ anchorname = '#' + ids[0]
+ num_entries[0] += 1
+ return anchorname
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_env_collector(TocTreeCollector)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/errors.py b/sphinx/errors.py
index 9ce41ec6f..c0339b4e9 100644
--- a/sphinx/errors.py
+++ b/sphinx/errors.py
@@ -1,5 +1,7 @@
"""Contains SphinxError and a few subclasses."""
+
from __future__ import annotations
+
from typing import Any
@@ -23,37 +25,49 @@ class SphinxError(Exception):
exception to a string ("category: message"). Should be set accordingly
in subclasses.
"""
+
category = 'Sphinx error'
class SphinxWarning(SphinxError):
"""Warning, treated as error."""
+
category = 'Warning, treated as error'
class ApplicationError(SphinxError):
"""Application initialization error."""
+
category = 'Application error'
class ExtensionError(SphinxError):
"""Extension error."""
- def __init__(self, message: str, orig_exc: (Exception | None)=None,
- modname: (str | None)=None) ->None:
+ def __init__(
+ self,
+ message: str,
+ orig_exc: Exception | None = None,
+ modname: str | None = None,
+ ) -> None:
super().__init__(message)
self.message = message
self.orig_exc = orig_exc
self.modname = modname
- def __repr__(self) ->str:
+ @property
+ def category(self) -> str: # type: ignore[override]
+ if self.modname:
+ return 'Extension error (%s)' % self.modname
+ else:
+ return 'Extension error'
+
+ def __repr__(self) -> str:
if self.orig_exc:
- return (
- f'{self.__class__.__name__}({self.message!r}, {self.orig_exc!r})'
- )
+ return f'{self.__class__.__name__}({self.message!r}, {self.orig_exc!r})'
return f'{self.__class__.__name__}({self.message!r})'
- def __str__(self) ->str:
+ def __str__(self) -> str:
parent_str = super().__str__()
if self.orig_exc:
return f'{parent_str} (exception: {self.orig_exc})'
@@ -62,45 +76,51 @@ class ExtensionError(SphinxError):
class BuildEnvironmentError(SphinxError):
"""BuildEnvironment error."""
+
category = 'BuildEnvironment error'
class ConfigError(SphinxError):
"""Configuration error."""
+
category = 'Configuration error'
class DocumentError(SphinxError):
"""Document error."""
+
category = 'Document error'
class ThemeError(SphinxError):
"""Theme error."""
+
category = 'Theme error'
class VersionRequirementError(SphinxError):
"""Incompatible Sphinx version error."""
+
category = 'Sphinx version error'
class SphinxParallelError(SphinxError):
"""Sphinx parallel build error."""
+
category = 'Sphinx parallel build error'
- def __init__(self, message: str, traceback: Any) ->None:
+ def __init__(self, message: str, traceback: Any) -> None:
self.message = message
self.traceback = traceback
- def __str__(self) ->str:
+ def __str__(self) -> str:
return self.message
class PycodeError(Exception):
"""Pycode Python source code analyser error."""
- def __str__(self) ->str:
+ def __str__(self) -> str:
res = self.args[0]
if len(self.args) > 1:
res += ' (exception was: %r)' % self.args[1]
@@ -111,9 +131,11 @@ class NoUri(Exception):
"""Raised by builder.get_relative_uri() or from missing-reference handlers
if there is no URI available.
"""
+
pass
class FiletypeNotFoundError(Exception):
"""Raised by get_filetype() if a filename matches no source suffix."""
+
pass
diff --git a/sphinx/events.py b/sphinx/events.py
index f40abbefa..17de456f0 100644
--- a/sphinx/events.py
+++ b/sphinx/events.py
@@ -2,19 +2,25 @@
Gracefully adapted from the TextPress system by Armin.
"""
+
from __future__ import annotations
+
from collections import defaultdict
from operator import attrgetter
from typing import TYPE_CHECKING, NamedTuple, overload
+
from sphinx.errors import ExtensionError, SphinxError
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.inspect import safe_getattr
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Sequence, Set
from pathlib import Path
from typing import Any, Literal
+
from docutils import nodes
+
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.builders import Builder
@@ -22,6 +28,8 @@ if TYPE_CHECKING:
from sphinx.domains import Domain
from sphinx.environment import BuildEnvironment
from sphinx.ext.todo import todo_node
+
+
logger = logging.getLogger(__name__)
@@ -31,49 +39,409 @@ class EventListener(NamedTuple):
priority: int
-core_events = {'config-inited': 'config', 'builder-inited': '',
+# List of all known core events. Maps name to arguments description.
+core_events = {
+ 'config-inited': 'config',
+ 'builder-inited': '',
'env-get-outdated': 'env, added, changed, removed',
- 'env-before-read-docs': 'env, docnames', 'env-purge-doc':
- 'env, docname', 'source-read': 'docname, source text', 'include-read':
- 'relative path, parent docname, source text', 'doctree-read':
- 'the doctree before being pickled', 'env-merge-info':
- 'env, read docnames, other env instance', 'env-updated': 'env',
- 'env-get-updated': 'env', 'env-check-consistency': 'env',
- 'write-started': 'builder', 'doctree-resolved': 'doctree, docname',
- 'missing-reference': 'env, node, contnode', 'warn-missing-reference':
- 'domain, node', 'build-finished': 'exception'}
+ 'env-before-read-docs': 'env, docnames',
+ 'env-purge-doc': 'env, docname',
+ 'source-read': 'docname, source text',
+ 'include-read': 'relative path, parent docname, source text',
+ 'doctree-read': 'the doctree before being pickled',
+ 'env-merge-info': 'env, read docnames, other env instance',
+ 'env-updated': 'env',
+ 'env-get-updated': 'env',
+ 'env-check-consistency': 'env',
+ 'write-started': 'builder',
+ 'doctree-resolved': 'doctree, docname',
+ 'missing-reference': 'env, node, contnode',
+ 'warn-missing-reference': 'domain, node',
+ 'build-finished': 'exception',
+}
class EventManager:
"""Event manager for Sphinx."""
- def __init__(self, app: Sphinx) ->None:
+ def __init__(self, app: Sphinx) -> None:
self.app = app
self.events = core_events.copy()
self.listeners: dict[str, list[EventListener]] = defaultdict(list)
self.next_listener_id = 0
- def add(self, name: str) ->None:
+ def add(self, name: str) -> None:
"""Register a custom Sphinx event."""
- pass
+ if name in self.events:
+ raise ExtensionError(__('Event %r already present') % name)
+ self.events[name] = ''
+
+ # ---- Core events -------------------------------------------------------
+
+ @overload
+ def connect(
+ self,
+ name: Literal['config-inited'],
+ callback: Callable[[Sphinx, Config], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['builder-inited'],
+ callback: Callable[[Sphinx], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['env-get-outdated'],
+ callback: Callable[
+ [Sphinx, BuildEnvironment, Set[str], Set[str], Set[str]], Sequence[str]
+ ],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['env-before-read-docs'],
+ callback: Callable[[Sphinx, BuildEnvironment, list[str]], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['env-purge-doc'],
+ callback: Callable[[Sphinx, BuildEnvironment, str], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['source-read'],
+ callback: Callable[[Sphinx, str, list[str]], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['include-read'],
+ callback: Callable[[Sphinx, Path, str, list[str]], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['doctree-read'],
+ callback: Callable[[Sphinx, nodes.document], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['env-merge-info'],
+ callback: Callable[
+ [Sphinx, BuildEnvironment, list[str], BuildEnvironment], None
+ ],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['env-updated'],
+ callback: Callable[[Sphinx, BuildEnvironment], str],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['env-get-updated'],
+ callback: Callable[[Sphinx, BuildEnvironment], Iterable[str]],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['env-check-consistency'],
+ callback: Callable[[Sphinx, BuildEnvironment], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['write-started'],
+ callback: Callable[[Sphinx, Builder], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['doctree-resolved'],
+ callback: Callable[[Sphinx, nodes.document, str], None],
+ priority: int,
+ ) -> int: ...
- def connect(self, name: str, callback: Callable, priority: int) ->int:
+ @overload
+ def connect(
+ self,
+ name: Literal['missing-reference'],
+ callback: Callable[
+ [Sphinx, BuildEnvironment, addnodes.pending_xref, nodes.TextElement],
+ nodes.reference | None,
+ ],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['warn-missing-reference'],
+ callback: Callable[[Sphinx, Domain, addnodes.pending_xref], bool | None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['build-finished'],
+ callback: Callable[[Sphinx, Exception | None], None],
+ priority: int,
+ ) -> int: ...
+
+ # ---- Events from builtin builders --------------------------------------
+
+ @overload
+ def connect(
+ self,
+ name: Literal['html-collect-pages'],
+ callback: Callable[[Sphinx], Iterable[tuple[str, dict[str, Any], str]]],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['html-page-context'],
+ callback: Callable[
+ [Sphinx, str, str, dict[str, Any], nodes.document], str | None
+ ],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['linkcheck-process-uri'],
+ callback: Callable[[Sphinx, str], str | None],
+ priority: int,
+ ) -> int: ...
+
+ # ---- Events from builtin extensions-- ----------------------------------
+
+ @overload
+ def connect(
+ self,
+ name: Literal['object-description-transform'],
+ callback: Callable[[Sphinx, str, str, addnodes.desc_content], None],
+ priority: int,
+ ) -> int: ...
+
+ # ---- Events from first-party extensions --------------------------------
+
+ @overload
+ def connect(
+ self,
+ name: Literal['autodoc-process-docstring'],
+ callback: Callable[
+ [
+ Sphinx,
+ Literal[
+ 'module', 'class', 'exception', 'function', 'method', 'attribute'
+ ],
+ str,
+ Any,
+ dict[str, bool],
+ Sequence[str],
+ ],
+ None,
+ ],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['autodoc-before-process-signature'],
+ callback: Callable[[Sphinx, Any, bool], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['autodoc-process-signature'],
+ callback: Callable[
+ [
+ Sphinx,
+ Literal[
+ 'module', 'class', 'exception', 'function', 'method', 'attribute'
+ ],
+ str,
+ Any,
+ dict[str, bool],
+ str | None,
+ str | None,
+ ],
+ tuple[str | None, str | None] | None,
+ ],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['autodoc-process-bases'],
+ callback: Callable[[Sphinx, str, Any, dict[str, bool], list[str]], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['autodoc-skip-member'],
+ callback: Callable[
+ [
+ Sphinx,
+ Literal[
+ 'module', 'class', 'exception', 'function', 'method', 'attribute'
+ ],
+ str,
+ Any,
+ bool,
+ dict[str, bool],
+ ],
+ bool,
+ ],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['todo-defined'],
+ callback: Callable[[Sphinx, todo_node], None],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['viewcode-find-source'],
+ callback: Callable[
+ [Sphinx, str],
+ tuple[str, dict[str, tuple[Literal['class', 'def', 'other'], int, int]]],
+ ],
+ priority: int,
+ ) -> int: ...
+
+ @overload
+ def connect(
+ self,
+ name: Literal['viewcode-follow-imported'],
+ callback: Callable[[Sphinx, str, str], str | None],
+ priority: int,
+ ) -> int: ...
+
+ # ---- Catch-all ---------------------------------------------------------
+
+ @overload
+ def connect(
+ self,
+ name: str,
+ callback: Callable[..., Any],
+ priority: int,
+ ) -> int: ...
+
+ def connect(self, name: str, callback: Callable, priority: int) -> int:
"""Connect a handler to specific event."""
- pass
+ if name not in self.events:
+ raise ExtensionError(__('Unknown event name: %s') % name)
- def disconnect(self, listener_id: int) ->None:
+ listener_id = self.next_listener_id
+ self.next_listener_id += 1
+ self.listeners[name].append(EventListener(listener_id, callback, priority))
+ return listener_id
+
+ def disconnect(self, listener_id: int) -> None:
"""Disconnect a handler."""
- pass
+ for listeners in self.listeners.values():
+ for listener in listeners.copy():
+ if listener.id == listener_id:
+ listeners.remove(listener)
- def emit(self, name: str, *args: Any, allowed_exceptions: tuple[type[
- Exception], ...]=()) ->list:
+ def emit(
+ self,
+ name: str,
+ *args: Any,
+ allowed_exceptions: tuple[type[Exception], ...] = (),
+ ) -> list:
"""Emit a Sphinx event."""
- pass
+ # not every object likes to be repr()'d (think
+ # random stuff coming via autodoc)
+ try:
+ repr_args = repr(args)
+ except Exception:
+ pass
+ else:
+ logger.debug('[app] emitting event: %r%s', name, repr_args)
+
+ results = []
+ listeners = sorted(self.listeners[name], key=attrgetter('priority'))
+ for listener in listeners:
+ try:
+ results.append(listener.handler(self.app, *args))
+ except allowed_exceptions:
+ # pass through the errors specified as *allowed_exceptions*
+ raise
+ except SphinxError:
+ raise
+ except Exception as exc:
+ if self.app.pdb:
+ # Just pass through the error, so that it can be debugged.
+ raise
+ modname = safe_getattr(listener.handler, '__module__', None)
+ raise ExtensionError(
+ __('Handler %r for event %r threw an exception')
+ % (listener.handler, name),
+ exc,
+ modname=modname,
+ ) from exc
+ return results
- def emit_firstresult(self, name: str, *args: Any, allowed_exceptions:
- tuple[type[Exception], ...]=()) ->Any:
+ def emit_firstresult(
+ self,
+ name: str,
+ *args: Any,
+ allowed_exceptions: tuple[type[Exception], ...] = (),
+ ) -> Any:
"""Emit a Sphinx event and returns first result.
This returns the result of the first handler that doesn't return ``None``.
"""
- pass
+ for result in self.emit(name, *args, allowed_exceptions=allowed_exceptions):
+ if result is not None:
+ return result
+ return None
diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py
index 255a1c96b..e08c0b1e9 100644
--- a/sphinx/ext/apidoc.py
+++ b/sphinx/ext/apidoc.py
@@ -8,7 +8,9 @@ This is derived from the "sphinx-autopackage" script, which is:
Copyright 2008 Société des arts technologiques (SAT),
https://sat.qc.ca/
"""
+
from __future__ import annotations
+
import argparse
import fnmatch
import glob
@@ -21,6 +23,7 @@ from importlib.machinery import EXTENSION_SUFFIXES
from os import path
from pathlib import Path
from typing import TYPE_CHECKING, Any, Protocol
+
import sphinx.locale
from sphinx import __display_version__, package_dir
from sphinx.cmd.quickstart import EXTENSIONS
@@ -28,109 +31,568 @@ from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.osutil import FileAvoidWrite, ensuredir
from sphinx.util.template import ReSTRenderer
+
if TYPE_CHECKING:
from collections.abc import Iterator, Sequence
+
logger = logging.getLogger(__name__)
+
+# automodule options
if 'SPHINX_APIDOC_OPTIONS' in os.environ:
OPTIONS = os.environ['SPHINX_APIDOC_OPTIONS'].split(',')
else:
- OPTIONS = ['members', 'undoc-members', 'show-inheritance']
-PY_SUFFIXES = '.py', '.pyx', *tuple(EXTENSION_SUFFIXES)
+ OPTIONS = [
+ 'members',
+ 'undoc-members',
+ # 'inherited-members', # disabled because there's a bug in sphinx
+ 'show-inheritance',
+ ]
+
+PY_SUFFIXES = ('.py', '.pyx', *tuple(EXTENSION_SUFFIXES))
+
template_dir = path.join(package_dir, 'templates', 'apidoc')
-def is_initpy(filename: (str | Path)) ->bool:
+def is_initpy(filename: str | Path) -> bool:
"""Check *filename* is __init__ file or not."""
- pass
+ basename = Path(filename).name
+ return any(
+ basename == '__init__' + suffix
+ for suffix in sorted(PY_SUFFIXES, key=len, reverse=True)
+ )
-def module_join(*modnames: (str | None)) ->str:
+def module_join(*modnames: str | None) -> str:
"""Join module names with dots."""
- pass
+ return '.'.join(filter(None, modnames))
-def is_packagedir(dirname: (str | None)=None, files: (list[str] | None)=None
- ) ->bool:
+def is_packagedir(dirname: str | None = None, files: list[str] | None = None) -> bool:
"""Check given *files* contains __init__ file."""
- pass
-
+ if files is dirname is None:
+ return False
-def write_file(name: str, text: str, opts: CliOptions) ->Path:
- """Write the output file for module/package <name>."""
- pass
+ if files is None:
+ files = os.listdir(dirname)
+ return any(f for f in files if is_initpy(f))
-def create_module_file(package: (str | None), basename: str, opts:
- CliOptions, user_template_dir: (str | None)=None) ->Path:
+def write_file(name: str, text: str, opts: CliOptions) -> Path:
+ """Write the output file for module/package <name>."""
+ fname = Path(opts.destdir, f'{name}.{opts.suffix}')
+ if opts.dryrun:
+ if not opts.quiet:
+ logger.info(__('Would create file %s.'), fname)
+ return fname
+ if not opts.force and fname.is_file():
+ if not opts.quiet:
+ logger.info(__('File %s already exists, skipping.'), fname)
+ else:
+ if not opts.quiet:
+ logger.info(__('Creating file %s.'), fname)
+ with FileAvoidWrite(fname) as f:
+ f.write(text)
+ return fname
+
+
+def create_module_file(
+ package: str | None,
+ basename: str,
+ opts: CliOptions,
+ user_template_dir: str | None = None,
+) -> Path:
"""Build the text of the file and write the file."""
- pass
-
-
-def create_package_file(root: str, master_package: (str | None), subroot:
- str, py_files: list[str], opts: CliOptions, subs: list[str],
- is_namespace: bool, excludes: Sequence[re.Pattern[str]]=(),
- user_template_dir: (str | None)=None) ->list[Path]:
+ options = copy(OPTIONS)
+ if opts.includeprivate and 'private-members' not in options:
+ options.append('private-members')
+
+ qualname = module_join(package, basename)
+ context = {
+ 'show_headings': not opts.noheadings,
+ 'basename': basename,
+ 'qualname': qualname,
+ 'automodule_options': options,
+ }
+ if user_template_dir is not None:
+ template_path = [user_template_dir, template_dir]
+ else:
+ template_path = [template_dir]
+ text = ReSTRenderer(template_path).render('module.rst.jinja', context)
+ return write_file(qualname, text, opts)
+
+
+def create_package_file(
+ root: str,
+ master_package: str | None,
+ subroot: str,
+ py_files: list[str],
+ opts: CliOptions,
+ subs: list[str],
+ is_namespace: bool,
+ excludes: Sequence[re.Pattern[str]] = (),
+ user_template_dir: str | None = None,
+) -> list[Path]:
"""Build the text of the file and write the file.
Also create submodules if necessary.
:returns: list of written files
"""
- pass
-
-
-def create_modules_toc_file(modules: list[str], opts: CliOptions, name: str
- ='modules', user_template_dir: (str | None)=None) ->Path:
+ # build a list of sub packages (directories containing an __init__ file)
+ subpackages = [
+ module_join(master_package, subroot, pkgname)
+ for pkgname in subs
+ if not is_skipped_package(Path(root, pkgname), opts, excludes)
+ ]
+ # build a list of sub modules
+ submodules = [
+ sub.split('.')[0]
+ for sub in py_files
+ if not is_skipped_module(Path(root, sub), opts, excludes) and not is_initpy(sub)
+ ]
+ submodules = sorted(set(submodules))
+ submodules = [
+ module_join(master_package, subroot, modname) for modname in submodules
+ ]
+ options = copy(OPTIONS)
+ if opts.includeprivate and 'private-members' not in options:
+ options.append('private-members')
+
+ pkgname = module_join(master_package, subroot)
+ context = {
+ 'pkgname': pkgname,
+ 'subpackages': subpackages,
+ 'submodules': submodules,
+ 'is_namespace': is_namespace,
+ 'modulefirst': opts.modulefirst,
+ 'separatemodules': opts.separatemodules,
+ 'automodule_options': options,
+ 'show_headings': not opts.noheadings,
+ 'maxdepth': opts.maxdepth,
+ }
+ if user_template_dir is not None:
+ template_path = [user_template_dir, template_dir]
+ else:
+ template_path = [template_dir]
+
+ written: list[Path] = []
+
+ text = ReSTRenderer(template_path).render('package.rst.jinja', context)
+ written.append(write_file(pkgname, text, opts))
+
+ if submodules and opts.separatemodules:
+ written.extend([
+ create_module_file(None, submodule, opts, user_template_dir)
+ for submodule in submodules
+ ])
+
+ return written
+
+
+def create_modules_toc_file(
+ modules: list[str],
+ opts: CliOptions,
+ name: str = 'modules',
+ user_template_dir: str | None = None,
+) -> Path:
"""Create the module's index."""
- pass
-
-
-def is_skipped_package(dirname: (str | Path), opts: CliOptions, excludes:
- Sequence[re.Pattern[str]]=()) ->bool:
+ modules.sort()
+ prev_module = ''
+ for module in modules.copy():
+ # look if the module is a subpackage and, if yes, ignore it
+ if module.startswith(prev_module + '.'):
+ modules.remove(module)
+ else:
+ prev_module = module
+
+ context = {
+ 'header': opts.header,
+ 'maxdepth': opts.maxdepth,
+ 'docnames': modules,
+ }
+ if user_template_dir is not None:
+ template_path = [user_template_dir, template_dir]
+ else:
+ template_path = [template_dir]
+ text = ReSTRenderer(template_path).render('toc.rst.jinja', context)
+ return write_file(name, text, opts)
+
+
+def is_skipped_package(
+ dirname: str | Path, opts: CliOptions, excludes: Sequence[re.Pattern[str]] = ()
+) -> bool:
"""Check if we want to skip this module."""
- pass
+ if not Path(dirname).is_dir():
+ return False
+ files = glob.glob(str(Path(dirname, '*.py')))
+ regular_package = any(f for f in files if is_initpy(f))
+ if not regular_package and not opts.implicit_namespaces:
+ # *dirname* is not both a regular package and an implicit namespace package
+ return True
-def is_skipped_module(filename: (str | Path), opts: CliOptions, _excludes:
- Sequence[re.Pattern[str]]) ->bool:
- """Check if we want to skip this module."""
- pass
+ # Check there is some showable module inside package
+ return all(is_excluded(Path(dirname, f), excludes) for f in files)
-def walk(rootpath: str, excludes: Sequence[re.Pattern[str]], opts: CliOptions
- ) ->Iterator[tuple[str, list[str], list[str]]]:
+def is_skipped_module(
+ filename: str | Path, opts: CliOptions, _excludes: Sequence[re.Pattern[str]]
+) -> bool:
+ """Check if we want to skip this module."""
+ filename = Path(filename)
+ if not filename.exists():
+ # skip if the file doesn't exist
+ return True
+ # skip if the module has a "private" name
+ return filename.name.startswith('_') and not opts.includeprivate
+
+
+def walk(
+ rootpath: str,
+ excludes: Sequence[re.Pattern[str]],
+ opts: CliOptions,
+) -> Iterator[tuple[str, list[str], list[str]]]:
"""Walk through the directory and list files and subdirectories up."""
- pass
-
-
-def has_child_module(rootpath: str, excludes: Sequence[re.Pattern[str]],
- opts: CliOptions) ->bool:
+ for root, subs, files in os.walk(rootpath, followlinks=opts.followlinks):
+ # document only Python module files (that aren't excluded)
+ files = sorted(
+ f
+ for f in files
+ if f.endswith(PY_SUFFIXES) and not is_excluded(Path(root, f), excludes)
+ )
+
+ # remove hidden ('.') and private ('_') directories, as well as
+ # excluded dirs
+ if opts.includeprivate:
+ exclude_prefixes: tuple[str, ...] = ('.',)
+ else:
+ exclude_prefixes = ('.', '_')
+
+ subs[:] = sorted(
+ sub
+ for sub in subs
+ if not sub.startswith(exclude_prefixes)
+ and not is_excluded(Path(root, sub), excludes)
+ )
+
+ yield root, subs, files
+
+
+def has_child_module(
+ rootpath: str, excludes: Sequence[re.Pattern[str]], opts: CliOptions
+) -> bool:
"""Check the given directory contains child module/s (at least one)."""
- pass
+ return any(files for _root, _subs, files in walk(rootpath, excludes, opts))
-def recurse_tree(rootpath: str, excludes: Sequence[re.Pattern[str]], opts:
- CliOptions, user_template_dir: (str | None)=None) ->tuple[list[Path],
- list[str]]:
+def recurse_tree(
+ rootpath: str,
+ excludes: Sequence[re.Pattern[str]],
+ opts: CliOptions,
+ user_template_dir: str | None = None,
+) -> tuple[list[Path], list[str]]:
"""
Look for every file in the directory tree and create the corresponding
ReST files.
"""
- pass
-
-
-def is_excluded(root: (str | Path), excludes: Sequence[re.Pattern[str]]
- ) ->bool:
+ # check if the base directory is a package and get its name
+ if is_packagedir(rootpath) or opts.implicit_namespaces:
+ root_package = rootpath.split(path.sep)[-1]
+ else:
+ # otherwise, the base is a directory with packages
+ root_package = None
+
+ toplevels = []
+ written_files = []
+ for root, subs, files in walk(rootpath, excludes, opts):
+ is_pkg = is_packagedir(None, files)
+ is_namespace = not is_pkg and opts.implicit_namespaces
+ if is_pkg:
+ for f in files.copy():
+ if is_initpy(f):
+ files.remove(f)
+ files.insert(0, f)
+ elif root != rootpath:
+ # only accept non-package at toplevel unless using implicit namespaces
+ if not opts.implicit_namespaces:
+ subs.clear()
+ continue
+
+ if is_pkg or is_namespace:
+ # we are in a package with something to document
+ if subs or len(files) > 1 or not is_skipped_package(root, opts):
+ subpackage = (
+ root[len(rootpath) :].lstrip(path.sep).replace(path.sep, '.')
+ )
+ # if this is not a namespace or
+ # a namespace and there is something there to document
+ if not is_namespace or has_child_module(root, excludes, opts):
+ written_files.extend(
+ create_package_file(
+ root,
+ root_package,
+ subpackage,
+ files,
+ opts,
+ subs,
+ is_namespace,
+ excludes,
+ user_template_dir,
+ )
+ )
+ toplevels.append(module_join(root_package, subpackage))
+ else:
+ # if we are at the root level, we don't require it to be a package
+ assert root == rootpath
+ assert root_package is None
+ for py_file in files:
+ if not is_skipped_module(Path(rootpath, py_file), opts, excludes):
+ module = py_file.split('.')[0]
+ written_files.append(
+ create_module_file(
+ root_package, module, opts, user_template_dir
+ )
+ )
+ toplevels.append(module)
+
+ return written_files, toplevels
+
+
+def is_excluded(root: str | Path, excludes: Sequence[re.Pattern[str]]) -> bool:
"""Check if the directory is in the exclude list.
Note: by having trailing slashes, we avoid common prefix issues, like
e.g. an exclude "foo" also accidentally excluding "foobar".
"""
- pass
+ root_str = str(root)
+ return any(exclude.match(root_str) for exclude in excludes)
+
+
+def get_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ usage='%(prog)s [OPTIONS] -o <OUTPUT_PATH> <MODULE_PATH> [EXCLUDE_PATTERN, ...]',
+ epilog=__('For more information, visit <https://www.sphinx-doc.org/>.'),
+ description=__("""
+Look recursively in <MODULE_PATH> for Python modules and packages and create
+one reST file with automodule directives per package in the <OUTPUT_PATH>.
+
+The <EXCLUDE_PATTERN>s can be file and/or directory patterns that will be
+excluded from generation.
+
+Note: By default this script will not overwrite already created files."""),
+ )
+
+ parser.add_argument(
+ '--version',
+ action='version',
+ dest='show_version',
+ version='%%(prog)s %s' % __display_version__,
+ )
+
+ parser.add_argument('module_path', help=__('path to module to document'))
+ parser.add_argument(
+ 'exclude_pattern',
+ nargs='*',
+ help=__(
+ 'fnmatch-style file and/or directory patterns to exclude from generation'
+ ),
+ )
+
+ parser.add_argument(
+ '-o',
+ '--output-dir',
+ action='store',
+ dest='destdir',
+ required=True,
+ help=__('directory to place all output'),
+ )
+ parser.add_argument(
+ '-q',
+ action='store_true',
+ dest='quiet',
+ help=__('no output on stdout, just warnings on stderr'),
+ )
+ parser.add_argument(
+ '-d',
+ '--maxdepth',
+ action='store',
+ dest='maxdepth',
+ type=int,
+ default=4,
+ help=__('maximum depth of submodules to show in the TOC (default: 4)'),
+ )
+ parser.add_argument(
+ '-f',
+ '--force',
+ action='store_true',
+ dest='force',
+ help=__('overwrite existing files'),
+ )
+ parser.add_argument(
+ '-l',
+ '--follow-links',
+ action='store_true',
+ dest='followlinks',
+ default=False,
+ help=__(
+ 'follow symbolic links. Powerful when combined with collective.recipe.omelette.'
+ ),
+ )
+ parser.add_argument(
+ '-n',
+ '--dry-run',
+ action='store_true',
+ dest='dryrun',
+ help=__('run the script without creating files'),
+ )
+ parser.add_argument(
+ '-e',
+ '--separate',
+ action='store_true',
+ dest='separatemodules',
+ help=__('put documentation for each module on its own page'),
+ )
+ parser.add_argument(
+ '-P',
+ '--private',
+ action='store_true',
+ dest='includeprivate',
+ help=__('include "_private" modules'),
+ )
+ parser.add_argument(
+ '--tocfile',
+ action='store',
+ dest='tocfile',
+ default='modules',
+ help=__('filename of table of contents (default: modules)'),
+ )
+ parser.add_argument(
+ '-T',
+ '--no-toc',
+ action='store_false',
+ dest='tocfile',
+ help=__("don't create a table of contents file"),
+ )
+ parser.add_argument(
+ '-E',
+ '--no-headings',
+ action='store_true',
+ dest='noheadings',
+ help=__(
+ "don't create headings for the module/package "
+ 'packages (e.g. when the docstrings already '
+ 'contain them)'
+ ),
+ )
+ parser.add_argument(
+ '-M',
+ '--module-first',
+ action='store_true',
+ dest='modulefirst',
+ help=__('put module documentation before submodule documentation'),
+ )
+ parser.add_argument(
+ '--implicit-namespaces',
+ action='store_true',
+ dest='implicit_namespaces',
+ help=__(
+ 'interpret module paths according to PEP-0420 implicit namespaces specification'
+ ),
+ )
+ parser.add_argument(
+ '-s',
+ '--suffix',
+ action='store',
+ dest='suffix',
+ default='rst',
+ help=__('file suffix (default: rst)'),
+ )
+ exclusive_group = parser.add_mutually_exclusive_group()
+ exclusive_group.add_argument(
+ '--remove-old',
+ action='store_true',
+ dest='remove_old',
+ help=__(
+ 'Remove existing files in the output directory that were not generated'
+ ),
+ )
+ exclusive_group.add_argument(
+ '-F',
+ '--full',
+ action='store_true',
+ dest='full',
+ help=__('generate a full project with sphinx-quickstart'),
+ )
+ parser.add_argument(
+ '-a',
+ '--append-syspath',
+ action='store_true',
+ dest='append_syspath',
+ help=__('append module_path to sys.path, used when --full is given'),
+ )
+ parser.add_argument(
+ '-H',
+ '--doc-project',
+ action='store',
+ dest='header',
+ help=__('project name (default: root module name)'),
+ )
+ parser.add_argument(
+ '-A',
+ '--doc-author',
+ action='store',
+ dest='author',
+ help=__('project author(s), used when --full is given'),
+ )
+ parser.add_argument(
+ '-V',
+ '--doc-version',
+ action='store',
+ dest='version',
+ help=__('project version, used when --full is given'),
+ )
+ parser.add_argument(
+ '-R',
+ '--doc-release',
+ action='store',
+ dest='release',
+ help=__(
+ 'project release, used when --full is given, defaults to --doc-version'
+ ),
+ )
+
+ group = parser.add_argument_group(__('extension options'))
+ group.add_argument(
+ '--extensions',
+ metavar='EXTENSIONS',
+ dest='extensions',
+ action='append',
+ help=__('enable arbitrary extensions'),
+ )
+ for ext in EXTENSIONS:
+ group.add_argument(
+ '--ext-%s' % ext,
+ action='append_const',
+ const='sphinx.ext.%s' % ext,
+ dest='extensions',
+ help=__('enable %s extension') % ext,
+ )
+
+ group = parser.add_argument_group(__('Project templating'))
+ group.add_argument(
+ '-t',
+ '--templatedir',
+ metavar='TEMPLATEDIR',
+ dest='templatedir',
+ help=__('template directory for template files'),
+ )
+
+ return parser
class CliOptions(Protocol):
"""Arguments parsed from the command line."""
+
module_path: str
exclude_pattern: list[str]
destdir: str
@@ -157,10 +619,104 @@ class CliOptions(Protocol):
remove_old: bool
-def main(argv: Sequence[str]=(), /) ->int:
+def main(argv: Sequence[str] = (), /) -> int:
"""Parse and check the command line arguments."""
- pass
-
-
+ locale.setlocale(locale.LC_ALL, '')
+ sphinx.locale.init_console()
+
+ parser = get_parser()
+ args: CliOptions = parser.parse_args(argv or sys.argv[1:])
+
+ rootpath = path.abspath(args.module_path)
+
+ # normalize opts
+
+ if args.header is None:
+ args.header = rootpath.split(path.sep)[-1]
+ if args.suffix.startswith('.'):
+ args.suffix = args.suffix[1:]
+ if not Path(rootpath).is_dir():
+ logger.error(__('%s is not a directory.'), rootpath)
+ raise SystemExit(1)
+ if not args.dryrun:
+ ensuredir(args.destdir)
+ excludes = tuple(
+ re.compile(fnmatch.translate(path.abspath(exclude)))
+ for exclude in dict.fromkeys(args.exclude_pattern)
+ )
+ written_files, modules = recurse_tree(rootpath, excludes, args, args.templatedir)
+
+ if args.full:
+ from sphinx.cmd import quickstart as qs
+
+ modules.sort()
+ prev_module = ''
+ text = ''
+ for module in modules:
+ if module.startswith(prev_module + '.'):
+ continue
+ prev_module = module
+ text += ' %s\n' % module
+ d: dict[str, Any] = {
+ 'path': args.destdir,
+ 'sep': False,
+ 'dot': '_',
+ 'project': args.header,
+ 'author': args.author or 'Author',
+ 'version': args.version or '',
+ 'release': args.release or args.version or '',
+ 'suffix': '.' + args.suffix,
+ 'master': 'index',
+ 'epub': True,
+ 'extensions': [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.viewcode',
+ 'sphinx.ext.todo',
+ ],
+ 'makefile': True,
+ 'batchfile': True,
+ 'make_mode': True,
+ 'mastertocmaxdepth': args.maxdepth,
+ 'mastertoctree': text,
+ 'language': 'en',
+ 'module_path': rootpath,
+ 'append_syspath': args.append_syspath,
+ }
+ if args.extensions:
+ d['extensions'].extend(args.extensions)
+ if args.quiet:
+ d['quiet'] = True
+
+ for ext in d['extensions'][:]:
+ if ',' in ext:
+ d['extensions'].remove(ext)
+ d['extensions'].extend(ext.split(','))
+
+ if not args.dryrun:
+ qs.generate(
+ d, silent=True, overwrite=args.force, templatedir=args.templatedir
+ )
+ elif args.tocfile:
+ written_files.append(
+ create_modules_toc_file(modules, args, args.tocfile, args.templatedir)
+ )
+
+ if args.remove_old and not args.dryrun:
+ for existing in Path(args.destdir).glob(f'**/*.{args.suffix}'):
+ if existing not in written_files:
+ try:
+ existing.unlink()
+ except OSError as exc:
+ logger.warning(
+ __('Failed to remove %s: %s'),
+ existing,
+ exc.strerror,
+ type='autodoc',
+ )
+
+ return 0
+
+
+# So program can be started with "python -m sphinx.apidoc ..."
if __name__ == '__main__':
raise SystemExit(main(sys.argv[1:]))
diff --git a/sphinx/ext/autodoc/directive.py b/sphinx/ext/autodoc/directive.py
index cbc59b16f..66b42a67d 100644
--- a/sphinx/ext/autodoc/directive.py
+++ b/sphinx/ext/autodoc/directive.py
@@ -1,43 +1,53 @@
from __future__ import annotations
+
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
+
from docutils import nodes
from docutils.statemachine import StringList
from docutils.utils import Reporter, assemble_option_dict
+
from sphinx.ext.autodoc import Documenter, Options
from sphinx.util import logging
from sphinx.util.docutils import SphinxDirective, switch_source_input
from sphinx.util.parsing import nested_parse_to_nodes
+
if TYPE_CHECKING:
from docutils.nodes import Node
from docutils.parsers.rst.states import RSTState
+
from sphinx.config import Config
from sphinx.environment import BuildEnvironment
+
logger = logging.getLogger(__name__)
+
+
+# common option names for autodoc directives
AUTODOC_DEFAULT_OPTIONS = ['members', 'undoc-members', 'inherited-members',
- 'show-inheritance', 'private-members', 'special-members',
- 'ignore-module-all', 'exclude-members', 'member-order',
- 'imported-members', 'class-doc-from', 'no-value']
-AUTODOC_EXTENDABLE_OPTIONS = ['members', 'private-members',
- 'special-members', 'exclude-members']
+ 'show-inheritance', 'private-members', 'special-members',
+ 'ignore-module-all', 'exclude-members', 'member-order',
+ 'imported-members', 'class-doc-from', 'no-value']
+
+AUTODOC_EXTENDABLE_OPTIONS = ['members', 'private-members', 'special-members',
+ 'exclude-members']
class DummyOptionSpec(dict[str, Callable[[str], str]]):
"""An option_spec allows any options."""
- def __bool__(self) ->bool:
+ def __bool__(self) -> bool:
"""Behaves like some options are defined."""
return True
- def __getitem__(self, _key: str) ->Callable[[str], str]:
+ def __getitem__(self, _key: str) -> Callable[[str], str]:
return lambda x: x
class DocumenterBridge:
"""A parameters container for Documenters."""
- def __init__(self, env: BuildEnvironment, reporter: (Reporter | None),
- options: Options, lineno: int, state: Any) ->None:
+ def __init__(self, env: BuildEnvironment, reporter: Reporter | None, options: Options,
+ lineno: int, state: Any) -> None:
self.env = env
self._reporter = reporter
self.genopt = options
@@ -47,16 +57,44 @@ class DocumenterBridge:
self.state = state
-def process_documenter_options(documenter: type[Documenter], config: Config,
- options: dict[str, str]) ->Options:
+def process_documenter_options(
+ documenter: type[Documenter], config: Config, options: dict[str, str],
+) -> Options:
"""Recognize options of Documenter from user input."""
- pass
+ default_options = config.autodoc_default_options
+ for name in AUTODOC_DEFAULT_OPTIONS:
+ if name not in documenter.option_spec:
+ continue
+ negated = options.pop('no-' + name, True) is None
+ if name in default_options and not negated:
+ if name in options and isinstance(default_options[name], str):
+ # take value from options if present or extend it
+ # with autodoc_default_options if necessary
+ if name in AUTODOC_EXTENDABLE_OPTIONS:
+ if options[name] is not None and options[name].startswith('+'):
+ options[name] = f'{default_options[name]},{options[name][1:]}'
+ else:
+ options[name] = default_options[name]
+
+ elif options.get(name) is not None:
+ # remove '+' from option argument if there's nothing to merge it with
+ options[name] = options[name].lstrip('+')
+
+ return Options(assemble_option_dict(options.items(), documenter.option_spec))
-def parse_generated_content(state: RSTState, content: StringList,
- documenter: Documenter) ->list[Node]:
+def parse_generated_content(state: RSTState, content: StringList, documenter: Documenter,
+ ) -> list[Node]:
"""Parse an item of content generated by Documenter."""
- pass
+ with switch_source_input(state, content):
+ if documenter.titles_allowed:
+ return nested_parse_to_nodes(state, content)
+
+ node = nodes.paragraph()
+ # necessary so that the child nodes get the right source/line set
+ node.document = state.document
+ state.nested_parse(content, 0, node, match_titles=False)
+ return node.children
class AutodocDirective(SphinxDirective):
@@ -65,8 +103,49 @@ class AutodocDirective(SphinxDirective):
It invokes a Documenter upon running. After the processing, it parses and returns
the content generated by Documenter.
"""
+
option_spec = DummyOptionSpec()
has_content = True
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
+
+ def run(self) -> list[Node]:
+ reporter = self.state.document.reporter
+
+ try:
+ source, lineno = reporter.get_source_and_line( # type: ignore[attr-defined]
+ self.lineno)
+ except AttributeError:
+ source, lineno = (None, None)
+ logger.debug('[autodoc] %s:%s: input:\n%s', source, lineno, self.block_text)
+
+ # look up target Documenter
+ objtype = self.name[4:] # strip prefix (auto-).
+ doccls = self.env.app.registry.documenters[objtype]
+
+ # process the options with the selected documenter's option_spec
+ try:
+ documenter_options = process_documenter_options(doccls, self.config, self.options)
+ except (KeyError, ValueError, TypeError) as exc:
+ # an option is either unknown or has a wrong type
+ logger.error('An option to %s is either unknown or has an invalid value: %s',
+ self.name, exc, location=(self.env.docname, lineno))
+ return []
+
+ # generate the output
+ params = DocumenterBridge(self.env, reporter, documenter_options, lineno, self.state)
+ documenter = doccls(params, self.arguments[0])
+ documenter.generate(more_content=self.content)
+ if not params.result:
+ return []
+
+ logger.debug('[autodoc] output:\n%s', '\n'.join(params.result))
+
+ # record all filenames as dependencies -- this will at least
+ # partially make automatic invalidation possible
+ for fn in params.record_dependencies:
+ self.state.document.settings.record_dependencies.add(fn)
+
+ result = parse_generated_content(self.state, params.result, documenter)
+ return result
diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py
index efb94f8f3..ebdaa9848 100644
--- a/sphinx/ext/autodoc/importer.py
+++ b/sphinx/ext/autodoc/importer.py
@@ -1,5 +1,7 @@
"""Importer utilities for autodoc"""
+
from __future__ import annotations
+
import contextlib
import importlib
import os
@@ -8,22 +10,36 @@ import traceback
import typing
from enum import Enum
from typing import TYPE_CHECKING, NamedTuple
+
from sphinx.errors import PycodeError
from sphinx.ext.autodoc.mock import ismock, undecorate
from sphinx.pycode import ModuleAnalyzer
from sphinx.util import logging
-from sphinx.util.inspect import getannotations, getmro, getslots, isclass, isenumclass, safe_getattr, unwrap_all
+from sphinx.util.inspect import (
+ getannotations,
+ getmro,
+ getslots,
+ isclass,
+ isenumclass,
+ safe_getattr,
+ unwrap_all,
+)
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator, Mapping
from types import ModuleType
from typing import Any
+
from sphinx.ext.autodoc import ObjectMember
+
logger = logging.getLogger(__name__)
-def _filter_enum_dict(enum_class: type[Enum], attrgetter: Callable[[Any,
- str, Any], Any], enum_class_dict: Mapping[str, object]) ->Iterator[tuple
- [str, type, Any]]:
+def _filter_enum_dict(
+ enum_class: type[Enum],
+ attrgetter: Callable[[Any, str, Any], Any],
+ enum_class_dict: Mapping[str, object],
+) -> Iterator[tuple[str, type, Any]]:
"""Find the attributes to document of an enumeration class.
The output consists of triplets ``(attribute name, defining class, value)``
@@ -31,29 +47,201 @@ def _filter_enum_dict(enum_class: type[Enum], attrgetter: Callable[[Any,
but with different defining class. The order of occurrence is guided by
the MRO of *enum_class*.
"""
- pass
+ # attributes that were found on a mixin type or the data type
+ candidate_in_mro: set[str] = set()
+ # sunder names that were picked up (and thereby allowed to be redefined)
+ # see: https://docs.python.org/3/howto/enum.html#supported-dunder-names
+ sunder_names = {'_name_', '_value_', '_missing_', '_order_', '_generate_next_value_'}
+ # attributes that can be picked up on a mixin type or the enum's data type
+ public_names = {'name', 'value', *object.__dict__, *sunder_names}
+ # names that are ignored by default
+ ignore_names = Enum.__dict__.keys() - public_names
+
+ def is_native_api(obj: object, name: str) -> bool:
+ """Check whether *obj* is the same as ``Enum.__dict__[name]``."""
+ return unwrap_all(obj) is unwrap_all(Enum.__dict__[name])
+
+ def should_ignore(name: str, value: Any) -> bool:
+ if name in sunder_names:
+ return is_native_api(value, name)
+ return name in ignore_names
+
+ sentinel = object()
+
+ def query(name: str, defining_class: type) -> tuple[str, type, Any] | None:
+ value = attrgetter(enum_class, name, sentinel)
+ if value is not sentinel:
+ return (name, defining_class, value)
+ return None
+
+ # attributes defined on a parent type, possibly shadowed later by
+ # the attributes defined directly inside the enumeration class
+ for parent in enum_class.__mro__:
+ if parent in {enum_class, Enum, object}:
+ continue
+
+ parent_dict = attrgetter(parent, '__dict__', {})
+ for name, value in parent_dict.items():
+ if should_ignore(name, value):
+ continue
+ candidate_in_mro.add(name)
+ if (item := query(name, parent)) is not None:
+ yield item
-def mangle(subject: Any, name: str) ->str:
+ # exclude members coming from the native Enum unless
+ # they were redefined on a mixin type or the data type
+ excluded_members = Enum.__dict__.keys() - candidate_in_mro
+ yield from filter(None, (query(name, enum_class) for name in enum_class_dict
+ if name not in excluded_members))
+
+ # check if allowed members from ``Enum`` were redefined at the enum level
+ special_names = sunder_names | public_names
+ special_names &= enum_class_dict.keys()
+ special_names &= Enum.__dict__.keys()
+ for name in special_names:
+ if (
+ not is_native_api(enum_class_dict[name], name)
+ and (item := query(name, enum_class)) is not None
+ ):
+ yield item
+
+
+def mangle(subject: Any, name: str) -> str:
"""Mangle the given name."""
- pass
+ try:
+ if isclass(subject) and name.startswith('__') and not name.endswith('__'):
+ return f"_{subject.__name__}{name}"
+ except AttributeError:
+ pass
+ return name
-def unmangle(subject: Any, name: str) ->(str | None):
+
+def unmangle(subject: Any, name: str) -> str | None:
"""Unmangle the given name."""
- pass
+ try:
+ if isclass(subject) and not name.endswith('__'):
+ prefix = "_%s__" % subject.__name__
+ if name.startswith(prefix):
+ return name.replace(prefix, "__", 1)
+ else:
+ for cls in subject.__mro__:
+ prefix = "_%s__" % cls.__name__
+ if name.startswith(prefix):
+ # mangled attribute defined in parent class
+ return None
+ except AttributeError:
+ pass
+
+ return name
-def import_module(modname: str) ->Any:
+def import_module(modname: str) -> Any:
"""Call importlib.import_module(modname), convert exceptions to ImportError."""
- pass
+ try:
+ return importlib.import_module(modname)
+ except BaseException as exc:
+ # Importing modules may cause any side effects, including
+ # SystemExit, so we need to catch all errors.
+ raise ImportError(exc, traceback.format_exc()) from exc
-def _reload_module(module: ModuleType) ->Any:
+def _reload_module(module: ModuleType) -> Any:
"""
Call importlib.reload(module), convert exceptions to ImportError
"""
- pass
+ try:
+ return importlib.reload(module)
+ except BaseException as exc:
+ # Importing modules may cause any side effects, including
+ # SystemExit, so we need to catch all errors.
+ raise ImportError(exc, traceback.format_exc()) from exc
+
+
+def import_object(modname: str, objpath: list[str], objtype: str = '',
+ attrgetter: Callable[[Any, str], Any] = safe_getattr) -> Any:
+ if objpath:
+ logger.debug('[autodoc] from %s import %s', modname, '.'.join(objpath))
+ else:
+ logger.debug('[autodoc] import %s', modname)
+
+ try:
+ module = None
+ exc_on_importing = None
+ objpath = objpath.copy()
+ while module is None:
+ try:
+ original_module_names = frozenset(sys.modules)
+ module = import_module(modname)
+ if os.environ.get('SPHINX_AUTODOC_RELOAD_MODULES'):
+ new_modules = [m for m in sys.modules if m not in original_module_names]
+ # Try reloading modules with ``typing.TYPE_CHECKING == True``.
+ try:
+ typing.TYPE_CHECKING = True
+ # Ignore failures; we've already successfully loaded these modules
+ with contextlib.suppress(ImportError, KeyError):
+ for m in new_modules:
+ _reload_module(sys.modules[m])
+ finally:
+ typing.TYPE_CHECKING = False
+ module = sys.modules[modname]
+ logger.debug('[autodoc] import %s => %r', modname, module)
+ except ImportError as exc:
+ logger.debug('[autodoc] import %s => failed', modname)
+ exc_on_importing = exc
+ if '.' in modname:
+ # retry with parent module
+ modname, name = modname.rsplit('.', 1)
+ objpath.insert(0, name)
+ else:
+ raise
+
+ obj = module
+ parent = None
+ object_name = None
+ for attrname in objpath:
+ parent = obj
+ logger.debug('[autodoc] getattr(_, %r)', attrname)
+ mangled_name = mangle(obj, attrname)
+ obj = attrgetter(obj, mangled_name)
+
+ try:
+ logger.debug('[autodoc] => %r', obj)
+ except TypeError:
+ # fallback of failure on logging for broken object
+ # refs: https://github.com/sphinx-doc/sphinx/issues/9095
+ logger.debug('[autodoc] => %r', (obj,))
+
+ object_name = attrname
+ return [module, parent, object_name, obj]
+ except (AttributeError, ImportError) as exc:
+ if isinstance(exc, AttributeError) and exc_on_importing:
+ # restore ImportError
+ exc = exc_on_importing
+
+ if objpath:
+ errmsg = ('autodoc: failed to import %s %r from module %r' %
+ (objtype, '.'.join(objpath), modname))
+ else:
+ errmsg = f'autodoc: failed to import {objtype} {modname!r}'
+
+ if isinstance(exc, ImportError):
+ # import_module() raises ImportError having real exception obj and
+ # traceback
+ real_exc, traceback_msg = exc.args
+ if isinstance(real_exc, SystemExit):
+ errmsg += ('; the module executes module level statement '
+ 'and it might call sys.exit().')
+ elif isinstance(real_exc, ImportError) and real_exc.args:
+ errmsg += '; the following exception was raised:\n%s' % real_exc.args[0]
+ else:
+ errmsg += '; the following exception was raised:\n%s' % traceback_msg
+ else:
+ errmsg += '; the following exception was raised:\n%s' % traceback.format_exc()
+
+ logger.debug(errmsg)
+ raise ImportError(errmsg) from exc
class Attribute(NamedTuple):
@@ -62,13 +250,157 @@ class Attribute(NamedTuple):
value: Any
-def get_object_members(subject: Any, objpath: list[str], attrgetter:
- Callable, analyzer: (ModuleAnalyzer | None)=None) ->dict[str, Attribute]:
+def get_object_members(
+ subject: Any,
+ objpath: list[str],
+ attrgetter: Callable,
+ analyzer: ModuleAnalyzer | None = None,
+) -> dict[str, Attribute]:
"""Get members and attributes of target object."""
- pass
+ from sphinx.ext.autodoc import INSTANCEATTR
+
+ # the members directly defined in the class
+ obj_dict = attrgetter(subject, '__dict__', {})
+
+ members: dict[str, Attribute] = {}
+
+ # enum members
+ if isenumclass(subject):
+ for name, defining_class, value in _filter_enum_dict(subject, attrgetter, obj_dict):
+ # the order of occurrence of *name* matches the subject's MRO,
+ # allowing inherited attributes to be shadowed correctly
+ if unmangled := unmangle(defining_class, name):
+ members[unmangled] = Attribute(unmangled, defining_class is subject, value)
+
+ # members in __slots__
+ try:
+ subject___slots__ = getslots(subject)
+ if subject___slots__:
+ from sphinx.ext.autodoc import SLOTSATTR
+
+ for name in subject___slots__:
+ members[name] = Attribute(name, True, SLOTSATTR)
+ except (TypeError, ValueError):
+ pass
+
+ # other members
+ for name in dir(subject):
+ try:
+ value = attrgetter(subject, name)
+ directly_defined = name in obj_dict
+ unmangled = unmangle(subject, name)
+ if unmangled and unmangled not in members:
+ members[unmangled] = Attribute(unmangled, directly_defined, value)
+ except AttributeError:
+ continue
+
+ # annotation only member (ex. attr: int)
+ for cls in getmro(subject):
+ for name in getannotations(cls):
+ unmangled = unmangle(cls, name)
+ if unmangled and unmangled not in members:
+ members[unmangled] = Attribute(unmangled, cls is subject, INSTANCEATTR)
+
+ if analyzer:
+ # append instance attributes (cf. self.attr1) if analyzer knows
+ namespace = '.'.join(objpath)
+ for (ns, name) in analyzer.find_attr_docs():
+ if namespace == ns and name not in members:
+ members[name] = Attribute(name, True, INSTANCEATTR)
+
+ return members
def get_class_members(subject: Any, objpath: Any, attrgetter: Callable,
- inherit_docstrings: bool=True) ->dict[str, ObjectMember]:
+ inherit_docstrings: bool = True) -> dict[str, ObjectMember]:
"""Get members and attributes of target class."""
- pass
+ from sphinx.ext.autodoc import INSTANCEATTR, ObjectMember
+
+ # the members directly defined in the class
+ obj_dict = attrgetter(subject, '__dict__', {})
+
+ members: dict[str, ObjectMember] = {}
+
+ # enum members
+ if isenumclass(subject):
+ for name, defining_class, value in _filter_enum_dict(subject, attrgetter, obj_dict):
+ # the order of occurrence of *name* matches the subject's MRO,
+ # allowing inherited attributes to be shadowed correctly
+ if unmangled := unmangle(defining_class, name):
+ members[unmangled] = ObjectMember(unmangled, value, class_=defining_class)
+
+ # members in __slots__
+ try:
+ subject___slots__ = getslots(subject)
+ if subject___slots__:
+ from sphinx.ext.autodoc import SLOTSATTR
+
+ for name, docstring in subject___slots__.items():
+ members[name] = ObjectMember(name, SLOTSATTR, class_=subject,
+ docstring=docstring)
+ except (TypeError, ValueError):
+ pass
+
+ # other members
+ for name in dir(subject):
+ try:
+ value = attrgetter(subject, name)
+ if ismock(value):
+ value = undecorate(value)
+
+ unmangled = unmangle(subject, name)
+ if unmangled and unmangled not in members:
+ if name in obj_dict:
+ members[unmangled] = ObjectMember(unmangled, value, class_=subject)
+ else:
+ members[unmangled] = ObjectMember(unmangled, value)
+ except AttributeError:
+ continue
+
+ try:
+ for cls in getmro(subject):
+ try:
+ modname = safe_getattr(cls, '__module__')
+ qualname = safe_getattr(cls, '__qualname__')
+ analyzer = ModuleAnalyzer.for_module(modname)
+ analyzer.analyze()
+ except AttributeError:
+ qualname = None
+ analyzer = None
+ except PycodeError:
+ analyzer = None
+
+ # annotation only member (ex. attr: int)
+ for name in getannotations(cls):
+ unmangled = unmangle(cls, name)
+ if unmangled and unmangled not in members:
+ if analyzer and (qualname, unmangled) in analyzer.attr_docs:
+ docstring = '\n'.join(analyzer.attr_docs[qualname, unmangled])
+ else:
+ docstring = None
+
+ members[unmangled] = ObjectMember(unmangled, INSTANCEATTR, class_=cls,
+ docstring=docstring)
+
+ # append or complete instance attributes (cf. self.attr1) if analyzer knows
+ if analyzer:
+ for (ns, name), docstring in analyzer.attr_docs.items():
+ if ns == qualname and name not in members:
+ # otherwise unknown instance attribute
+ members[name] = ObjectMember(name, INSTANCEATTR, class_=cls,
+ docstring='\n'.join(docstring))
+ elif (ns == qualname and docstring and
+ isinstance(members[name], ObjectMember) and
+ not members[name].docstring):
+ if cls != subject and not inherit_docstrings:
+ # If we are in the MRO of the class and not the class itself,
+ # and we do not want to inherit docstrings, then skip setting
+ # the docstring below
+ continue
+ # attribute is already known, because dir(subject) enumerates it.
+ # But it has no docstring yet
+ members[name].docstring = '\n'.join(docstring)
+ except AttributeError:
+ pass
+
+ return members
diff --git a/sphinx/ext/autodoc/mock.py b/sphinx/ext/autodoc/mock.py
index 2a6922a24..265f45057 100644
--- a/sphinx/ext/autodoc/mock.py
+++ b/sphinx/ext/autodoc/mock.py
@@ -1,5 +1,7 @@
"""mock for autodoc"""
+
from __future__ import annotations
+
import contextlib
import os
import sys
@@ -7,124 +9,195 @@ from importlib.abc import Loader, MetaPathFinder
from importlib.machinery import ModuleSpec
from types import MethodType, ModuleType
from typing import TYPE_CHECKING
+
from sphinx.util import logging
from sphinx.util.inspect import isboundmethod, safe_getattr
+
if TYPE_CHECKING:
from collections.abc import Iterator, Sequence
from typing import Any
+
from typing_extensions import TypeIs
+
logger = logging.getLogger(__name__)
class _MockObject:
"""Used by autodoc_mock_imports."""
+
__display_name__ = '_MockObject'
__name__ = ''
__sphinx_mock__ = True
__sphinx_decorator_args__: tuple[Any, ...] = ()
- def __new__(cls, *args: Any, **kwargs: Any) ->Any:
+ def __new__(cls, *args: Any, **kwargs: Any) -> Any:
if len(args) == 3 and isinstance(args[1], tuple):
superclass = args[1][-1].__class__
if superclass is cls:
+ # subclassing MockObject
return _make_subclass(args[0], superclass.__display_name__,
- superclass=superclass, attributes=args[2])
+ superclass=superclass, attributes=args[2])
+
return super().__new__(cls)
- def __init__(self, *args: Any, **kwargs: Any) ->None:
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
self.__qualname__ = self.__name__
- def __len__(self) ->int:
+ def __len__(self) -> int:
return 0
- def __contains__(self, key: str) ->bool:
+ def __contains__(self, key: str) -> bool:
return False
- def __iter__(self) ->Iterator[Any]:
+ def __iter__(self) -> Iterator[Any]:
return iter(())
- def __mro_entries__(self, bases: tuple[Any, ...]) ->tuple[type, ...]:
- return self.__class__,
+ def __mro_entries__(self, bases: tuple[Any, ...]) -> tuple[type, ...]:
+ return (self.__class__,)
- def __getitem__(self, key: Any) ->_MockObject:
- return _make_subclass(str(key), self.__display_name__, self.__class__)(
- )
+ def __getitem__(self, key: Any) -> _MockObject:
+ return _make_subclass(str(key), self.__display_name__, self.__class__)()
- def __getattr__(self, key: str) ->_MockObject:
+ def __getattr__(self, key: str) -> _MockObject:
return _make_subclass(key, self.__display_name__, self.__class__)()
- def __call__(self, *args: Any, **kwargs: Any) ->Any:
+ def __call__(self, *args: Any, **kwargs: Any) -> Any:
call = self.__class__()
call.__sphinx_decorator_args__ = args
return call
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return self.__display_name__
+def _make_subclass(name: str, module: str, superclass: Any = _MockObject,
+ attributes: Any = None, decorator_args: tuple[Any, ...] = ()) -> Any:
+ attrs = {'__module__': module,
+ '__display_name__': module + '.' + name,
+ '__name__': name,
+ '__sphinx_decorator_args__': decorator_args}
+ attrs.update(attributes or {})
+
+ return type(name, (superclass,), attrs)
+
+
class _MockModule(ModuleType):
"""Used by autodoc_mock_imports."""
+
__file__ = os.devnull
__sphinx_mock__ = True
- def __init__(self, name: str) ->None:
+ def __init__(self, name: str) -> None:
super().__init__(name)
self.__all__: list[str] = []
self.__path__: list[str] = []
- def __getattr__(self, name: str) ->_MockObject:
+ def __getattr__(self, name: str) -> _MockObject:
return _make_subclass(name, self.__name__)()
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return self.__name__
class MockLoader(Loader):
"""A loader for mocking."""
- def __init__(self, finder: MockFinder) ->None:
+ def __init__(self, finder: MockFinder) -> None:
super().__init__()
self.finder = finder
+ def create_module(self, spec: ModuleSpec) -> ModuleType:
+ logger.debug('[autodoc] adding a mock module as %s!', spec.name)
+ self.finder.mocked_modules.append(spec.name)
+ return _MockModule(spec.name)
+
+ def exec_module(self, module: ModuleType) -> None:
+ pass # nothing to do
+
class MockFinder(MetaPathFinder):
"""A finder for mocking."""
- def __init__(self, modnames: list[str]) ->None:
+ def __init__(self, modnames: list[str]) -> None:
super().__init__()
self.modnames = modnames
self.loader = MockLoader(self)
self.mocked_modules: list[str] = []
- def invalidate_caches(self) ->None:
+ def find_spec(self, fullname: str, path: Sequence[bytes | str] | None,
+ target: ModuleType | None = None) -> ModuleSpec | None:
+ for modname in self.modnames:
+ # check if fullname is (or is a descendant of) one of our targets
+ if modname == fullname or fullname.startswith(modname + '.'):
+ return ModuleSpec(fullname, self.loader)
+
+ return None
+
+ def invalidate_caches(self) -> None:
"""Invalidate mocked modules on sys.modules."""
- pass
+ for modname in self.mocked_modules:
+ sys.modules.pop(modname, None)
@contextlib.contextmanager
-def mock(modnames: list[str]) ->Iterator[None]:
+def mock(modnames: list[str]) -> Iterator[None]:
"""Insert mock modules during context::
with mock(['target.module.name']):
# mock modules are enabled here
...
"""
- pass
+ finder = MockFinder(modnames)
+ try:
+ sys.meta_path.insert(0, finder)
+ yield
+ finally:
+ sys.meta_path.remove(finder)
+ finder.invalidate_caches()
-def ismockmodule(subject: Any) ->TypeIs[_MockModule]:
+def ismockmodule(subject: Any) -> TypeIs[_MockModule]:
"""Check if the object is a mocked module."""
- pass
+ return isinstance(subject, _MockModule)
-def ismock(subject: Any) ->bool:
+def ismock(subject: Any) -> bool:
"""Check if the object is mocked."""
- pass
+ # check the object has '__sphinx_mock__' attribute
+ try:
+ if safe_getattr(subject, '__sphinx_mock__', None) is None:
+ return False
+ except AttributeError:
+ return False
+
+ # check the object is mocked module
+ if isinstance(subject, _MockModule):
+ return True
+
+ # check the object is bound method
+ if isinstance(subject, MethodType) and isboundmethod(subject):
+ tmp_subject = subject.__func__
+ else:
+ tmp_subject = subject
+
+ try:
+ # check the object is mocked object
+ __mro__ = safe_getattr(type(tmp_subject), '__mro__', [])
+ if len(__mro__) > 2 and __mro__[-2] is _MockObject:
+ # A mocked object has a MRO that ends with (..., _MockObject, object).
+ return True
+ except AttributeError:
+ pass
+
+ return False
-def undecorate(subject: _MockObject) ->Any:
+def undecorate(subject: _MockObject) -> Any:
"""Unwrap mock if *subject* is decorated by mocked object.
If not decorated, returns given *subject* itself.
"""
- pass
+ if ismock(subject) and subject.__sphinx_decorator_args__:
+ return subject.__sphinx_decorator_args__[0]
+ else:
+ return subject
diff --git a/sphinx/ext/autodoc/preserve_defaults.py b/sphinx/ext/autodoc/preserve_defaults.py
index fc27cde55..824293424 100644
--- a/sphinx/ext/autodoc/preserve_defaults.py
+++ b/sphinx/ext/autodoc/preserve_defaults.py
@@ -3,52 +3,198 @@
Preserve the default argument values of function signatures in source code
and keep them not evaluated for readability.
"""
+
from __future__ import annotations
+
import ast
import inspect
import types
import warnings
from typing import TYPE_CHECKING
+
import sphinx
from sphinx.deprecation import RemovedInSphinx90Warning
from sphinx.locale import __
from sphinx.pycode.ast import unparse as ast_unparse
from sphinx.util import logging
+
if TYPE_CHECKING:
from typing import Any
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
-_LAMBDA_NAME = (lambda : None).__name__
+_LAMBDA_NAME = (lambda: None).__name__
class DefaultValue:
-
- def __init__(self, name: str) ->None:
+ def __init__(self, name: str) -> None:
self.name = name
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return self.name
-def get_function_def(obj: Any) ->(ast.FunctionDef | None):
+def get_function_def(obj: Any) -> ast.FunctionDef | None:
"""Get FunctionDef object from living object.
This tries to parse original code for living object and returns
AST node for given *obj*.
"""
- pass
+ warnings.warn('sphinx.ext.autodoc.preserve_defaults.get_function_def is'
+ ' deprecated and scheduled for removal in Sphinx 9.'
+ ' Use sphinx.ext.autodoc.preserve_defaults._get_arguments() to'
+ ' extract AST arguments objects from a lambda or regular'
+ ' function.', RemovedInSphinx90Warning, stacklevel=2)
+
+ try:
+ source = inspect.getsource(obj)
+ if source.startswith((' ', '\t')):
+ # subject is placed inside class or block. To read its docstring,
+ # this adds if-block before the declaration.
+ module = ast.parse('if True:\n' + source)
+ return module.body[0].body[0] # type: ignore[attr-defined]
+ else:
+ module = ast.parse(source)
+ return module.body[0] # type: ignore[return-value]
+ except (OSError, TypeError): # failed to load source code
+ return None
-def _get_arguments(obj: Any, /) ->(ast.arguments | None):
+def _get_arguments(obj: Any, /) -> ast.arguments | None:
"""Parse 'ast.arguments' from an object.
This tries to parse the original code for an object and returns
an 'ast.arguments' node.
"""
- pass
+ try:
+ source = inspect.getsource(obj)
+ if source.startswith((' ', '\t')):
+ # 'obj' is in some indented block.
+ module = ast.parse('if True:\n' + source)
+ subject = module.body[0].body[0] # type: ignore[attr-defined]
+ else:
+ module = ast.parse(source)
+ subject = module.body[0]
+ except (OSError, TypeError):
+ # bail; failed to load source for 'obj'.
+ return None
+ except SyntaxError:
+ if _is_lambda(obj):
+ # Most likely a multi-line arising from detecting a lambda, e.g.:
+ #
+ # class Egg:
+ # x = property(
+ # lambda self: 1, doc="...")
+ return None
+
+ # Other syntax errors that are not due to the fact that we are
+ # documenting a lambda function are propagated
+ # (in particular if a lambda is renamed by the user).
+ raise
+
+ return _get_arguments_inner(subject)
+
+
+def _is_lambda(x: Any, /) -> bool:
+ return isinstance(x, types.LambdaType) and x.__name__ == _LAMBDA_NAME
-def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) ->None:
+def _get_arguments_inner(x: Any, /) -> ast.arguments | None:
+ if isinstance(x, ast.AsyncFunctionDef | ast.FunctionDef | ast.Lambda):
+ return x.args
+ if isinstance(x, ast.Assign | ast.AnnAssign):
+ return _get_arguments_inner(x.value)
+ return None
+
+
+def get_default_value(lines: list[str], position: ast.expr) -> str | None:
+ try:
+ if position.lineno == position.end_lineno:
+ line = lines[position.lineno - 1]
+ return line[position.col_offset:position.end_col_offset]
+ else:
+ # multiline value is not supported now
+ return None
+ except (AttributeError, IndexError):
+ return None
+
+
+def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) -> None:
"""Update defvalue info of *obj* using type_comments."""
- pass
+ if not app.config.autodoc_preserve_defaults:
+ return
+
+ try:
+ lines = inspect.getsource(obj).splitlines()
+ if lines[0].startswith((' ', '\t')):
+ # insert a dummy line to follow what _get_arguments() does.
+ lines.insert(0, '')
+ except (OSError, TypeError):
+ lines = []
+
+ try:
+ args = _get_arguments(obj)
+ except SyntaxError:
+ return
+ if args is None:
+ # If the object is a built-in, we won't be always able to recover
+ # the function definition and its arguments. This happens if *obj*
+ # is the `__init__` method generated automatically for dataclasses.
+ return
+
+ if not args.defaults and not args.kw_defaults:
+ return
+
+ try:
+ if bound_method and inspect.ismethod(obj) and hasattr(obj, '__func__'):
+ sig = inspect.signature(obj.__func__)
+ else:
+ sig = inspect.signature(obj)
+ defaults = list(args.defaults)
+ kw_defaults = list(args.kw_defaults)
+ parameters = list(sig.parameters.values())
+ for i, param in enumerate(parameters):
+ if param.default is param.empty:
+ if param.kind == param.KEYWORD_ONLY:
+ # Consume kw_defaults for kwonly args
+ kw_defaults.pop(0)
+ else:
+ if param.kind in (param.POSITIONAL_ONLY, param.POSITIONAL_OR_KEYWORD):
+ default = defaults.pop(0)
+ value = get_default_value(lines, default)
+ if value is None:
+ value = ast_unparse(default)
+ parameters[i] = param.replace(default=DefaultValue(value))
+ else:
+ default = kw_defaults.pop(0) # type: ignore[assignment]
+ value = get_default_value(lines, default)
+ if value is None:
+ value = ast_unparse(default)
+ parameters[i] = param.replace(default=DefaultValue(value))
+
+ sig = sig.replace(parameters=parameters)
+ try:
+ obj.__signature__ = sig
+ except AttributeError:
+ # __signature__ can't be set directly on bound methods.
+ obj.__dict__['__signature__'] = sig
+ except (AttributeError, TypeError):
+ # Failed to update signature (e.g. built-in or extension types).
+ # For user-defined functions, "obj" may not have __dict__,
+ # e.g. when decorated with a class that defines __slots__.
+ # In this case, we can't set __signature__.
+ return
+ except NotImplementedError as exc: # failed to ast_unparse()
+ logger.warning(__("Failed to parse a default argument value for %r: %s"), obj, exc)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_config_value('autodoc_preserve_defaults', False, 'env')
+ app.connect('autodoc-before-process-signature', update_defvalue)
+
+ return {
+ 'version': sphinx.__display_version__,
+ 'parallel_read_safe': True,
+ }
diff --git a/sphinx/ext/autodoc/type_comment.py b/sphinx/ext/autodoc/type_comment.py
index 545663871..e0a5a63b9 100644
--- a/sphinx/ext/autodoc/type_comment.py
+++ b/sphinx/ext/autodoc/type_comment.py
@@ -1,43 +1,141 @@
"""Update annotations info of living objects using type_comments."""
+
from __future__ import annotations
+
import ast
from inspect import Parameter, Signature, getsource
from typing import TYPE_CHECKING, Any, cast
+
import sphinx
from sphinx.locale import __
from sphinx.pycode.ast import unparse as ast_unparse
from sphinx.util import inspect, logging
+
if TYPE_CHECKING:
from collections.abc import Sequence
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
-def not_suppressed(argtypes: Sequence[ast.expr]=()) ->bool:
+def not_suppressed(argtypes: Sequence[ast.expr] = ()) -> bool:
"""Check given *argtypes* is suppressed type_comment or not."""
- pass
+ if len(argtypes) == 0: # no argtypees
+ return False
+ if len(argtypes) == 1:
+ arg = argtypes[0]
+ if isinstance(arg, ast.Constant) and arg.value is ...: # suppressed
+ return False
+ # not suppressed
+ return True
def signature_from_ast(node: ast.FunctionDef, bound_method: bool,
- type_comment: ast.FunctionDef) ->Signature:
+ type_comment: ast.FunctionDef) -> Signature:
"""Return a Signature object for the given *node*.
:param bound_method: Specify *node* is a bound method or not
"""
- pass
+ params = []
+ for arg in node.args.posonlyargs:
+ param = Parameter(arg.arg, Parameter.POSITIONAL_ONLY, annotation=arg.type_comment)
+ params.append(param)
+
+ for arg in node.args.args:
+ param = Parameter(arg.arg, Parameter.POSITIONAL_OR_KEYWORD,
+ annotation=arg.type_comment or Parameter.empty)
+ params.append(param)
+
+ if node.args.vararg:
+ param = Parameter(node.args.vararg.arg, Parameter.VAR_POSITIONAL,
+ annotation=node.args.vararg.type_comment or Parameter.empty)
+ params.append(param)
+
+ for arg in node.args.kwonlyargs:
+ param = Parameter(arg.arg, Parameter.KEYWORD_ONLY,
+ annotation=arg.type_comment or Parameter.empty)
+ params.append(param)
+ if node.args.kwarg:
+ param = Parameter(node.args.kwarg.arg, Parameter.VAR_KEYWORD,
+ annotation=node.args.kwarg.type_comment or Parameter.empty)
+ params.append(param)
-def get_type_comment(obj: Any, bound_method: bool=False) ->(Signature | None):
+ # Remove first parameter when *obj* is bound_method
+ if bound_method and params:
+ params.pop(0)
+
+ # merge type_comment into signature
+ if not_suppressed(type_comment.argtypes): # type: ignore[attr-defined]
+ for i, param in enumerate(params):
+ params[i] = param.replace(
+ annotation=type_comment.argtypes[i]) # type: ignore[attr-defined]
+
+ if node.returns:
+ return Signature(params, return_annotation=node.returns)
+ elif type_comment.returns:
+ return Signature(params, return_annotation=ast_unparse(type_comment.returns))
+ else:
+ return Signature(params)
+
+
+def get_type_comment(obj: Any, bound_method: bool = False) -> Signature | None:
"""Get type_comment'ed FunctionDef object from living object.
This tries to parse original code for living object and returns
Signature for given *obj*.
"""
- pass
+ try:
+ source = getsource(obj)
+ if source.startswith((' ', r'\t')):
+ # subject is placed inside class or block. To read its docstring,
+ # this adds if-block before the declaration.
+ module = ast.parse('if True:\n' + source, type_comments=True)
+ subject = cast(
+ ast.FunctionDef, module.body[0].body[0], # type: ignore[attr-defined]
+ )
+ else:
+ module = ast.parse(source, type_comments=True)
+ subject = cast(ast.FunctionDef, module.body[0])
+
+ type_comment = getattr(subject, "type_comment", None)
+ if type_comment:
+ function = ast.parse(type_comment, mode='func_type', type_comments=True)
+ return signature_from_ast(
+ subject, bound_method, function, # type: ignore[arg-type]
+ )
+ else:
+ return None
+ except (OSError, TypeError): # failed to load source code
+ return None
+ except SyntaxError: # failed to parse type_comments
+ return None
-def update_annotations_using_type_comments(app: Sphinx, obj: Any,
- bound_method: bool) ->None:
+def update_annotations_using_type_comments(app: Sphinx, obj: Any, bound_method: bool) -> None:
"""Update annotations info of *obj* using type_comments."""
- pass
+ try:
+ type_sig = get_type_comment(obj, bound_method)
+ if type_sig:
+ sig = inspect.signature(obj, bound_method)
+ for param in sig.parameters.values():
+ if param.name not in obj.__annotations__:
+ annotation = type_sig.parameters[param.name].annotation
+ if annotation is not Parameter.empty:
+ obj.__annotations__[param.name] = ast_unparse(annotation)
+
+ if 'return' not in obj.__annotations__:
+ obj.__annotations__['return'] = type_sig.return_annotation
+ except KeyError as exc:
+ logger.warning(__("Failed to update signature for %r: parameter not found: %s"),
+ obj, exc)
+ except NotImplementedError as exc: # failed to ast.unparse()
+ logger.warning(__("Failed to parse type_comment for %r: %s"), obj, exc)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.connect('autodoc-before-process-signature', update_annotations_using_type_comments)
+
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/autodoc/typehints.py b/sphinx/ext/autodoc/typehints.py
index 97ed01df9..ed8860cb7 100644
--- a/sphinx/ext/autodoc/typehints.py
+++ b/sphinx/ext/autodoc/typehints.py
@@ -1,20 +1,220 @@
"""Generating content for autodoc using typehints"""
+
from __future__ import annotations
+
import re
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes
+
import sphinx
from sphinx import addnodes
from sphinx.util import inspect
from sphinx.util.typing import ExtensionMetadata, stringify_annotation
+
if TYPE_CHECKING:
from docutils.nodes import Element
+
from sphinx.application import Sphinx
from sphinx.ext.autodoc import Options
def record_typehints(app: Sphinx, objtype: str, name: str, obj: Any,
- options: Options, args: str, retann: str) ->None:
+ options: Options, args: str, retann: str) -> None:
"""Record type hints to env object."""
- pass
+ if app.config.autodoc_typehints_format == 'short':
+ mode = 'smart'
+ else:
+ mode = 'fully-qualified'
+
+ try:
+ if callable(obj):
+ annotations = app.env.temp_data.setdefault('annotations', {})
+ annotation = annotations.setdefault(name, {})
+ sig = inspect.signature(obj, type_aliases=app.config.autodoc_type_aliases)
+ for param in sig.parameters.values():
+ if param.annotation is not param.empty:
+ annotation[param.name] = stringify_annotation(param.annotation, mode) # type: ignore[arg-type]
+ if sig.return_annotation is not sig.empty:
+ annotation['return'] = stringify_annotation(sig.return_annotation, mode) # type: ignore[arg-type]
+ except (TypeError, ValueError):
+ pass
+
+
+def merge_typehints(app: Sphinx, domain: str, objtype: str, contentnode: Element) -> None:
+ if domain != 'py':
+ return
+ if app.config.autodoc_typehints not in ('both', 'description'):
+ return
+
+ try:
+ signature = cast(addnodes.desc_signature, contentnode.parent[0])
+ if signature['module']:
+ fullname = f'{signature["module"]}.{signature["fullname"]}'
+ else:
+ fullname = signature['fullname']
+ except KeyError:
+ # signature node does not have valid context info for the target object
+ return
+
+ annotations = app.env.temp_data.get('annotations', {})
+ if annotations.get(fullname, {}):
+ field_lists = [n for n in contentnode if isinstance(n, nodes.field_list)]
+ if field_lists == []:
+ field_list = insert_field_list(contentnode)
+ field_lists.append(field_list)
+
+ for field_list in field_lists:
+ if app.config.autodoc_typehints_description_target == "all":
+ if objtype == 'class':
+ modify_field_list(field_list, annotations[fullname], suppress_rtype=True)
+ else:
+ modify_field_list(field_list, annotations[fullname])
+ elif app.config.autodoc_typehints_description_target == "documented_params":
+ augment_descriptions_with_types(
+ field_list, annotations[fullname], force_rtype=True,
+ )
+ else:
+ augment_descriptions_with_types(
+ field_list, annotations[fullname], force_rtype=False,
+ )
+
+
+def insert_field_list(node: Element) -> nodes.field_list:
+ field_list = nodes.field_list()
+ desc = [n for n in node if isinstance(n, addnodes.desc)]
+ if desc:
+ # insert just before sub object descriptions (ex. methods, nested classes, etc.)
+ index = node.index(desc[0])
+ node.insert(index - 1, [field_list])
+ else:
+ node += field_list
+
+ return field_list
+
+
+def modify_field_list(node: nodes.field_list, annotations: dict[str, str],
+ suppress_rtype: bool = False) -> None:
+ arguments: dict[str, dict[str, bool]] = {}
+ fields = cast(Iterable[nodes.field], node)
+ for field in fields:
+ field_name = field[0].astext()
+ parts = re.split(' +', field_name)
+ if parts[0] == 'param':
+ if len(parts) == 2:
+ # :param xxx:
+ arg = arguments.setdefault(parts[1], {})
+ arg['param'] = True
+ elif len(parts) > 2:
+ # :param xxx yyy:
+ name = ' '.join(parts[2:])
+ arg = arguments.setdefault(name, {})
+ arg['param'] = True
+ arg['type'] = True
+ elif parts[0] == 'type':
+ name = ' '.join(parts[1:])
+ arg = arguments.setdefault(name, {})
+ arg['type'] = True
+ elif parts[0] == 'rtype':
+ arguments['return'] = {'type': True}
+
+ for name, annotation in annotations.items():
+ if name == 'return':
+ continue
+
+ if '*' + name in arguments:
+ name = '*' + name
+ arguments.get(name)
+ elif '**' + name in arguments:
+ name = '**' + name
+ arguments.get(name)
+ else:
+ arg = arguments.get(name, {})
+
+ if not arg.get('type'):
+ field = nodes.field()
+ field += nodes.field_name('', 'type ' + name)
+ field += nodes.field_body('', nodes.paragraph('', annotation))
+ node += field
+ if not arg.get('param'):
+ field = nodes.field()
+ field += nodes.field_name('', 'param ' + name)
+ field += nodes.field_body('', nodes.paragraph('', ''))
+ node += field
+
+ if 'return' in annotations and 'return' not in arguments:
+ annotation = annotations['return']
+ if annotation == 'None' and suppress_rtype:
+ return
+
+ field = nodes.field()
+ field += nodes.field_name('', 'rtype')
+ field += nodes.field_body('', nodes.paragraph('', annotation))
+ node += field
+
+
+def augment_descriptions_with_types(
+ node: nodes.field_list,
+ annotations: dict[str, str],
+ force_rtype: bool,
+) -> None:
+ fields = cast(Iterable[nodes.field], node)
+ has_description: set[str] = set()
+ has_type: set[str] = set()
+ for field in fields:
+ field_name = field[0].astext()
+ parts = re.split(' +', field_name)
+ if parts[0] == 'param':
+ if len(parts) == 2:
+ # :param xxx:
+ has_description.add(parts[1])
+ elif len(parts) > 2:
+ # :param xxx yyy:
+ name = ' '.join(parts[2:])
+ has_description.add(name)
+ has_type.add(name)
+ elif parts[0] == 'type':
+ name = ' '.join(parts[1:])
+ has_type.add(name)
+ elif parts[0] in ('return', 'returns'):
+ has_description.add('return')
+ elif parts[0] == 'rtype':
+ has_type.add('return')
+
+ # Add 'type' for parameters with a description but no declared type.
+ for name, annotation in annotations.items():
+ if name in ('return', 'returns'):
+ continue
+
+ if '*' + name in has_description:
+ name = '*' + name
+ elif '**' + name in has_description:
+ name = '**' + name
+
+ if name in has_description and name not in has_type:
+ field = nodes.field()
+ field += nodes.field_name('', 'type ' + name)
+ field += nodes.field_body('', nodes.paragraph('', annotation))
+ node += field
+
+ # Add 'rtype' if 'return' is present and 'rtype' isn't.
+ if 'return' in annotations:
+ rtype = annotations['return']
+ if 'return' not in has_type and ('return' in has_description or
+ (force_rtype and rtype != "None")):
+ field = nodes.field()
+ field += nodes.field_name('', 'rtype')
+ field += nodes.field_body('', nodes.paragraph('', rtype))
+ node += field
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.connect('autodoc-process-signature', record_typehints)
+ app.connect('object-description-transform', merge_typehints)
+
+ return {
+ 'version': sphinx.__display_version__,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/ext/autosectionlabel.py b/sphinx/ext/autosectionlabel.py
index c617cb79e..c1eb46bf0 100644
--- a/sphinx/ext/autosectionlabel.py
+++ b/sphinx/ext/autosectionlabel.py
@@ -1,14 +1,70 @@
"""Allow reference sections by :ref: role using its title."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, cast
+
from docutils import nodes
+
import sphinx
from sphinx.domains.std import StandardDomain
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.nodes import clean_astext
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
+
+
+def get_node_depth(node: Node) -> int:
+ i = 0
+ cur_node = node
+ while cur_node.parent != node.document:
+ cur_node = cur_node.parent
+ i += 1
+ return i
+
+
+def register_sections_as_label(app: Sphinx, document: Node) -> None:
+ domain = cast(StandardDomain, app.env.get_domain('std'))
+ for node in document.findall(nodes.section):
+ if (app.config.autosectionlabel_maxdepth and
+ get_node_depth(node) >= app.config.autosectionlabel_maxdepth):
+ continue
+ labelid = node['ids'][0]
+ docname = app.env.docname
+ title = cast(nodes.title, node[0])
+ ref_name = getattr(title, 'rawsource', title.astext())
+ if app.config.autosectionlabel_prefix_document:
+ name = nodes.fully_normalize_name(docname + ':' + ref_name)
+ else:
+ name = nodes.fully_normalize_name(ref_name)
+ sectname = clean_astext(title)
+
+ logger.debug(__('section "%s" gets labeled as "%s"'),
+ ref_name, name,
+ location=node, type='autosectionlabel', subtype=docname)
+ if name in domain.labels:
+ logger.warning(__('duplicate label %s, other instance in %s'),
+ name, app.env.doc2path(domain.labels[name][0]),
+ location=node, type='autosectionlabel', subtype=docname)
+
+ domain.anonlabels[name] = docname, labelid
+ domain.labels[name] = docname, labelid, sectname
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_config_value('autosectionlabel_prefix_document', False, 'env')
+ app.add_config_value('autosectionlabel_maxdepth', None, 'env')
+ app.connect('doctree-read', register_sections_as_label)
+
+ return {
+ 'version': sphinx.__display_version__,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py
index eb7b16602..b6f31be9b 100644
--- a/sphinx/ext/autosummary/generate.py
+++ b/sphinx/ext/autosummary/generate.py
@@ -11,7 +11,9 @@ Example Makefile rule::
generate:
sphinx-autogen -o source/generated source/*.rst
"""
+
from __future__ import annotations
+
import argparse
import importlib
import inspect
@@ -24,15 +26,22 @@ import sys
from os import path
from pathlib import Path
from typing import TYPE_CHECKING, Any, NamedTuple
+
from jinja2 import TemplateNotFound
from jinja2.sandbox import SandboxedEnvironment
+
import sphinx.locale
from sphinx import __display_version__, package_dir
from sphinx.builders import Builder
from sphinx.config import Config
from sphinx.errors import PycodeError
from sphinx.ext.autodoc.importer import import_module
-from sphinx.ext.autosummary import ImportExceptionGroup, get_documenter, import_by_name, import_ivar_by_name
+from sphinx.ext.autosummary import (
+ ImportExceptionGroup,
+ get_documenter,
+ import_by_name,
+ import_ivar_by_name,
+)
from sphinx.locale import __
from sphinx.pycode import ModuleAnalyzer
from sphinx.registry import SphinxComponentRegistry
@@ -40,18 +49,21 @@ from sphinx.util import logging, rst
from sphinx.util.inspect import getall, safe_getattr
from sphinx.util.osutil import ensuredir
from sphinx.util.template import SphinxTemplateLoader
+
if TYPE_CHECKING:
from collections.abc import Sequence, Set
from gettext import NullTranslations
+
from sphinx.application import Sphinx
from sphinx.ext.autodoc import Documenter
+
logger = logging.getLogger(__name__)
class DummyApplication:
"""Dummy Application class for sphinx-autogen command."""
- def __init__(self, translator: NullTranslations) ->None:
+ def __init__(self, translator: NullTranslations) -> None:
self.config = Config()
self.registry = SphinxComponentRegistry()
self.messagelog: list[str] = []
@@ -60,10 +72,14 @@ class DummyApplication:
self.verbosity = 0
self._warncount = 0
self._exception_on_warning = False
+
self.config.add('autosummary_context', {}, 'env', ())
self.config.add('autosummary_filename_map', {}, 'env', ())
self.config.add('autosummary_ignore_module_all', True, 'env', bool)
+ def emit_firstresult(self, *args: Any) -> None:
+ pass
+
class AutosummaryEntry(NamedTuple):
name: str
@@ -72,31 +88,82 @@ class AutosummaryEntry(NamedTuple):
recursive: bool
+def setup_documenters(app: Any) -> None:
+ from sphinx.ext.autodoc import (
+ AttributeDocumenter,
+ ClassDocumenter,
+ DataDocumenter,
+ DecoratorDocumenter,
+ ExceptionDocumenter,
+ FunctionDocumenter,
+ MethodDocumenter,
+ ModuleDocumenter,
+ PropertyDocumenter,
+ )
+
+ documenters: list[type[Documenter]] = [
+ ModuleDocumenter,
+ ClassDocumenter,
+ ExceptionDocumenter,
+ DataDocumenter,
+ FunctionDocumenter,
+ MethodDocumenter,
+ AttributeDocumenter,
+ DecoratorDocumenter,
+ PropertyDocumenter,
+ ]
+ for documenter in documenters:
+ app.registry.add_documenter(documenter.objtype, documenter)
+
+
+def _underline(title: str, line: str = '=') -> str:
+ if '\n' in title:
+ msg = 'Can only underline single lines'
+ raise ValueError(msg)
+ return title + '\n' + line * len(title)
+
+
class AutosummaryRenderer:
"""A helper class for rendering."""
- def __init__(self, app: Sphinx) ->None:
+ def __init__(self, app: Sphinx) -> None:
if isinstance(app, Builder):
msg = 'Expected a Sphinx application object!'
raise ValueError(msg)
- system_templates_path = [os.path.join(package_dir, 'ext',
- 'autosummary', 'templates')]
- loader = SphinxTemplateLoader(app.srcdir, app.config.templates_path,
- system_templates_path)
+
+ system_templates_path = [
+ os.path.join(package_dir, 'ext', 'autosummary', 'templates')
+ ]
+ loader = SphinxTemplateLoader(
+ app.srcdir, app.config.templates_path, system_templates_path
+ )
+
self.env = SandboxedEnvironment(loader=loader)
self.env.filters['escape'] = rst.escape
self.env.filters['e'] = rst.escape
self.env.filters['underline'] = _underline
+
if app.translator:
self.env.add_extension('jinja2.ext.i18n')
- self.env.install_gettext_translations(app.translator)
+ # ``install_gettext_translations`` is injected by the ``jinja2.ext.i18n`` extension
+ self.env.install_gettext_translations(app.translator) # type: ignore[attr-defined]
- def render(self, template_name: str, context: dict[str, Any]) ->str:
+ def render(self, template_name: str, context: dict[str, Any]) -> str:
"""Render a template file."""
- pass
+ try:
+ template = self.env.get_template(template_name)
+ except TemplateNotFound:
+ try:
+ # objtype is given as template_name
+ template = self.env.get_template('autosummary/%s.rst' % template_name)
+ except TemplateNotFound:
+ # fallback to base.rst
+ template = self.env.get_template('autosummary/base.rst')
+
+ return template.render(context)
-def _split_full_qualified_name(name: str) ->tuple[str | None, str]:
+def _split_full_qualified_name(name: str) -> tuple[str | None, str]:
"""Split full qualified name to a pair of modname and qualname.
A qualname is an abbreviation for "Qualified name" introduced at PEP-3155
@@ -110,59 +177,500 @@ def _split_full_qualified_name(name: str) ->tuple[str | None, str]:
Therefore you need to mock 3rd party modules if needed before
calling this function.
"""
- pass
+ parts = name.split('.')
+ for i, _part in enumerate(parts, 1):
+ try:
+ modname = '.'.join(parts[:i])
+ importlib.import_module(modname)
+ except ImportError:
+ if parts[: i - 1]:
+ return '.'.join(parts[: i - 1]), '.'.join(parts[i - 1 :])
+ else:
+ return None, '.'.join(parts)
+ except IndexError:
+ pass
+ return name, ''
-class ModuleScanner:
- def __init__(self, app: Any, obj: Any) ->None:
+# -- Generating output ---------------------------------------------------------
+
+
+class ModuleScanner:
+ def __init__(self, app: Any, obj: Any) -> None:
self.app = app
self.object = obj
-
-def members_of(obj: Any, conf: Config) ->Sequence[str]:
+ def get_object_type(self, name: str, value: Any) -> str:
+ return get_documenter(self.app, value, self.object).objtype
+
+ def is_skipped(self, name: str, value: Any, objtype: str) -> bool:
+ try:
+ return self.app.emit_firstresult(
+ 'autodoc-skip-member', objtype, name, value, False, {}
+ )
+ except Exception as exc:
+ logger.warning(
+ __(
+ 'autosummary: failed to determine %r to be documented, '
+ 'the following exception was raised:\n%s'
+ ),
+ name,
+ exc,
+ type='autosummary',
+ )
+ return False
+
+ def scan(self, imported_members: bool) -> list[str]:
+ members = []
+ try:
+ analyzer = ModuleAnalyzer.for_module(self.object.__name__)
+ attr_docs = analyzer.find_attr_docs()
+ except PycodeError:
+ attr_docs = {}
+
+ for name in members_of(self.object, self.app.config):
+ try:
+ value = safe_getattr(self.object, name)
+ except AttributeError:
+ value = None
+
+ objtype = self.get_object_type(name, value)
+ if self.is_skipped(name, value, objtype):
+ continue
+
+ try:
+ if ('', name) in attr_docs:
+ imported = False
+ elif inspect.ismodule(value): # NoQA: SIM114
+ imported = True
+ elif safe_getattr(value, '__module__') != self.object.__name__:
+ imported = True
+ else:
+ imported = False
+ except AttributeError:
+ imported = False
+
+ respect_module_all = not self.app.config.autosummary_ignore_module_all
+ if (
+ # list all members up
+ imported_members
+ # list not-imported members
+ or imported is False
+ # list members that have __all__ set
+ or (respect_module_all and '__all__' in dir(self.object))
+ ):
+ members.append(name)
+
+ return members
+
+
+def members_of(obj: Any, conf: Config) -> Sequence[str]:
"""Get the members of ``obj``, possibly ignoring the ``__all__`` module attribute
Follows the ``conf.autosummary_ignore_module_all`` setting.
"""
- pass
-
-
-def _get_module_attrs(name: str, members: Any) ->tuple[list[str], list[str]]:
+ if conf.autosummary_ignore_module_all:
+ return dir(obj)
+ else:
+ return getall(obj) or dir(obj)
+
+
+def generate_autosummary_content(
+ name: str,
+ obj: Any,
+ parent: Any,
+ template: AutosummaryRenderer,
+ template_name: str,
+ imported_members: bool,
+ app: Any,
+ recursive: bool,
+ context: dict[str, Any],
+ modname: str | None = None,
+ qualname: str | None = None,
+) -> str:
+ doc = get_documenter(app, obj, parent)
+
+ ns: dict[str, Any] = {}
+ ns.update(context)
+
+ if doc.objtype == 'module':
+ scanner = ModuleScanner(app, obj)
+ ns['members'] = scanner.scan(imported_members)
+
+ respect_module_all = not app.config.autosummary_ignore_module_all
+ imported_members = imported_members or (
+ '__all__' in dir(obj) and respect_module_all
+ )
+
+ ns['functions'], ns['all_functions'] = _get_members(
+ doc, app, obj, {'function'}, imported=imported_members
+ )
+ ns['classes'], ns['all_classes'] = _get_members(
+ doc, app, obj, {'class'}, imported=imported_members
+ )
+ ns['exceptions'], ns['all_exceptions'] = _get_members(
+ doc, app, obj, {'exception'}, imported=imported_members
+ )
+ ns['attributes'], ns['all_attributes'] = _get_module_attrs(name, ns['members'])
+ ispackage = hasattr(obj, '__path__')
+ if ispackage and recursive:
+ # Use members that are not modules as skip list, because it would then mean
+ # that module was overwritten in the package namespace
+ skip = (
+ ns['all_functions']
+ + ns['all_classes']
+ + ns['all_exceptions']
+ + ns['all_attributes']
+ )
+
+ # If respect_module_all and module has a __all__ attribute, first get
+ # modules that were explicitly imported. Next, find the rest with the
+ # get_modules method, but only put in "public" modules that are in the
+ # __all__ list
+ #
+ # Otherwise, use get_modules method normally
+ if respect_module_all and '__all__' in dir(obj):
+ imported_modules, all_imported_modules = _get_members(
+ doc, app, obj, {'module'}, imported=True
+ )
+ skip += all_imported_modules
+ public_members = getall(obj)
+ else:
+ imported_modules, all_imported_modules = [], []
+ public_members = None
+
+ modules, all_modules = _get_modules(
+ obj, skip=skip, name=name, public_members=public_members
+ )
+ ns['modules'] = imported_modules + modules
+ ns['all_modules'] = all_imported_modules + all_modules
+ elif doc.objtype == 'class':
+ ns['members'] = dir(obj)
+ ns['inherited_members'] = set(dir(obj)) - set(obj.__dict__.keys())
+ ns['methods'], ns['all_methods'] = _get_members(
+ doc, app, obj, {'method'}, include_public={'__init__'}
+ )
+ ns['attributes'], ns['all_attributes'] = _get_members(
+ doc, app, obj, {'attribute', 'property'}
+ )
+
+ if modname is None or qualname is None:
+ modname, qualname = _split_full_qualified_name(name)
+
+ if doc.objtype in ('method', 'attribute', 'property'):
+ ns['class'] = qualname.rsplit('.', 1)[0]
+
+ if doc.objtype == 'class':
+ shortname = qualname
+ else:
+ shortname = qualname.rsplit('.', 1)[-1]
+
+ ns['fullname'] = name
+ ns['module'] = modname
+ ns['objname'] = qualname
+ ns['name'] = shortname
+
+ ns['objtype'] = doc.objtype
+ ns['underline'] = len(name) * '='
+
+ if template_name:
+ return template.render(template_name, ns)
+ else:
+ return template.render(doc.objtype, ns)
+
+
+def _skip_member(app: Sphinx, obj: Any, name: str, objtype: str) -> bool:
+ try:
+ return app.emit_firstresult(
+ 'autodoc-skip-member', objtype, name, obj, False, {}
+ )
+ except Exception as exc:
+ logger.warning(
+ __(
+ 'autosummary: failed to determine %r to be documented, '
+ 'the following exception was raised:\n%s'
+ ),
+ name,
+ exc,
+ type='autosummary',
+ )
+ return False
+
+
+def _get_class_members(obj: Any) -> dict[str, Any]:
+ members = sphinx.ext.autodoc.importer.get_class_members(obj, None, safe_getattr)
+ return {name: member.object for name, member in members.items()}
+
+
+def _get_module_members(app: Sphinx, obj: Any) -> dict[str, Any]:
+ members = {}
+ for name in members_of(obj, app.config):
+ try:
+ members[name] = safe_getattr(obj, name)
+ except AttributeError:
+ continue
+ return members
+
+
+def _get_all_members(doc: type[Documenter], app: Sphinx, obj: Any) -> dict[str, Any]:
+ if doc.objtype == 'module':
+ return _get_module_members(app, obj)
+ elif doc.objtype == 'class':
+ return _get_class_members(obj)
+ return {}
+
+
+def _get_members(
+ doc: type[Documenter],
+ app: Sphinx,
+ obj: Any,
+ types: set[str],
+ *,
+ include_public: Set[str] = frozenset(),
+ imported: bool = True,
+) -> tuple[list[str], list[str]]:
+ items: list[str] = []
+ public: list[str] = []
+
+ all_members = _get_all_members(doc, app, obj)
+ for name, value in all_members.items():
+ documenter = get_documenter(app, value, obj)
+ if documenter.objtype in types:
+ # skip imported members if expected
+ if imported or getattr(value, '__module__', None) == obj.__name__:
+ skipped = _skip_member(app, value, name, documenter.objtype)
+ if skipped is True:
+ pass
+ elif skipped is False:
+ # show the member forcedly
+ items.append(name)
+ public.append(name)
+ else:
+ items.append(name)
+ if name in include_public or not name.startswith('_'):
+ # considers member as public
+ public.append(name)
+ return public, items
+
+
+def _get_module_attrs(name: str, members: Any) -> tuple[list[str], list[str]]:
"""Find module attributes with docstrings."""
- pass
-
-
-def generate_autosummary_docs(sources: list[str], output_dir: (str | os.
- PathLike[str] | None)=None, suffix: str='.rst', base_path: (str | os.
- PathLike[str] | None)=None, imported_members: bool=False, app: (Sphinx |
- None)=None, overwrite: bool=True, encoding: str='utf-8') ->list[Path]:
+ attrs, public = [], []
+ try:
+ analyzer = ModuleAnalyzer.for_module(name)
+ attr_docs = analyzer.find_attr_docs()
+ for namespace, attr_name in attr_docs:
+ if namespace == '' and attr_name in members:
+ attrs.append(attr_name)
+ if not attr_name.startswith('_'):
+ public.append(attr_name)
+ except PycodeError:
+ pass # give up if ModuleAnalyzer fails to parse code
+ return public, attrs
+
+
+def _get_modules(
+ obj: Any,
+ *,
+ skip: Sequence[str],
+ name: str,
+ public_members: Sequence[str] | None = None,
+) -> tuple[list[str], list[str]]:
+ items: list[str] = []
+ public: list[str] = []
+ for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__):
+ if modname in skip:
+ # module was overwritten in __init__.py, so not accessible
+ continue
+ fullname = f'{name}.{modname}'
+ try:
+ module = import_module(fullname)
+ except ImportError:
+ pass
+ else:
+ if module and hasattr(module, '__sphinx_mock__'):
+ continue
+
+ items.append(modname)
+ if public_members is not None:
+ if modname in public_members:
+ public.append(modname)
+ else:
+ if not modname.startswith('_'):
+ public.append(modname)
+ return public, items
+
+
+def generate_autosummary_docs(
+ sources: list[str],
+ output_dir: str | os.PathLike[str] | None = None,
+ suffix: str = '.rst',
+ base_path: str | os.PathLike[str] | None = None,
+ imported_members: bool = False,
+ app: Sphinx | None = None,
+ overwrite: bool = True,
+ encoding: str = 'utf-8',
+) -> list[Path]:
"""Generate autosummary documentation for the given sources.
:returns: list of generated files (both new and existing ones)
"""
- pass
-
-
-def find_autosummary_in_files(filenames: list[str]) ->list[AutosummaryEntry]:
+ assert app is not None, 'app is required'
+
+ showed_sources = sorted(sources)
+ if len(showed_sources) > 20:
+ showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:]
+ logger.info(
+ __('[autosummary] generating autosummary for: %s'), ', '.join(showed_sources)
+ )
+
+ if output_dir:
+ logger.info(__('[autosummary] writing to %s'), output_dir)
+
+ if base_path is not None:
+ sources = [os.path.join(base_path, filename) for filename in sources]
+
+ template = AutosummaryRenderer(app)
+
+ # read
+ items = find_autosummary_in_files(sources)
+
+ # keep track of new files
+ new_files: list[Path] = []
+ all_files: list[Path] = []
+
+ filename_map = app.config.autosummary_filename_map
+
+ # write
+ for entry in sorted(set(items), key=str):
+ if entry.path is None:
+ # The corresponding autosummary:: directive did not have
+ # a :toctree: option
+ continue
+
+ path = output_dir or os.path.abspath(entry.path)
+ ensuredir(path)
+
+ try:
+ name, obj, parent, modname = import_by_name(entry.name)
+ qualname = name.replace(modname + '.', '')
+ except ImportExceptionGroup as exc:
+ try:
+ # try to import as an instance attribute
+ name, obj, parent, modname = import_ivar_by_name(entry.name)
+ qualname = name.replace(modname + '.', '')
+ except ImportError as exc2:
+ if exc2.__cause__:
+ exceptions: list[BaseException] = [*exc.exceptions, exc2.__cause__]
+ else:
+ exceptions = [*exc.exceptions, exc2]
+
+ errors = list({f'* {type(e).__name__}: {e}' for e in exceptions})
+ logger.warning(
+ __('[autosummary] failed to import %s.\nPossible hints:\n%s'),
+ entry.name,
+ '\n'.join(errors),
+ )
+ continue
+
+ context: dict[str, Any] = {**app.config.autosummary_context}
+
+ content = generate_autosummary_content(
+ name,
+ obj,
+ parent,
+ template,
+ entry.template,
+ imported_members,
+ app,
+ entry.recursive,
+ context,
+ modname,
+ qualname,
+ )
+
+ file_path = Path(path, filename_map.get(name, name) + suffix)
+ all_files.append(file_path)
+ if file_path.is_file():
+ with file_path.open(encoding=encoding) as f:
+ old_content = f.read()
+
+ if content == old_content:
+ continue
+ if overwrite: # content has changed
+ with file_path.open('w', encoding=encoding) as f:
+ f.write(content)
+ new_files.append(file_path)
+ else:
+ with open(file_path, 'w', encoding=encoding) as f:
+ f.write(content)
+ new_files.append(file_path)
+
+ # descend recursively to new files
+ if new_files:
+ all_files.extend(
+ generate_autosummary_docs(
+ [str(f) for f in new_files],
+ output_dir=output_dir,
+ suffix=suffix,
+ base_path=base_path,
+ imported_members=imported_members,
+ app=app,
+ overwrite=overwrite,
+ )
+ )
+
+ return all_files
+
+
+# -- Finding documented entries in files ---------------------------------------
+
+
+def find_autosummary_in_files(filenames: list[str]) -> list[AutosummaryEntry]:
"""Find out what items are documented in source/*.rst.
See `find_autosummary_in_lines`.
"""
- pass
-
-
-def find_autosummary_in_docstring(name: str, filename: (str | None)=None
- ) ->list[AutosummaryEntry]:
+ documented: list[AutosummaryEntry] = []
+ for filename in filenames:
+ with open(filename, encoding='utf-8', errors='ignore') as f:
+ lines = f.read().splitlines()
+ documented.extend(find_autosummary_in_lines(lines, filename=filename))
+ return documented
+
+
+def find_autosummary_in_docstring(
+ name: str,
+ filename: str | None = None,
+) -> list[AutosummaryEntry]:
"""Find out what items are documented in the given object's docstring.
See `find_autosummary_in_lines`.
"""
- pass
-
-
-def find_autosummary_in_lines(lines: list[str], module: (str | None)=None,
- filename: (str | None)=None) ->list[AutosummaryEntry]:
+ try:
+ real_name, obj, parent, modname = import_by_name(name)
+ lines = pydoc.getdoc(obj).splitlines()
+ return find_autosummary_in_lines(lines, module=name, filename=filename)
+ except AttributeError:
+ pass
+ except ImportExceptionGroup as exc:
+ errors = '\n'.join({f'* {type(e).__name__}: {e}' for e in exc.exceptions})
+ logger.warning(f'Failed to import {name}.\nPossible hints:\n{errors}') # NoQA: G004
+ except SystemExit:
+ logger.warning(
+ "Failed to import '%s'; the module executes module level "
+ 'statement and it might call sys.exit().',
+ name,
+ )
+ return []
+
+
+def find_autosummary_in_lines(
+ lines: list[str],
+ module: str | None = None,
+ filename: str | None = None,
+) -> list[AutosummaryEntry]:
"""Find out what items appear in autosummary:: directives in the
given lines.
@@ -173,7 +681,203 @@ def find_autosummary_in_lines(lines: list[str], module: (str | None)=None,
*template* ``None`` if the directive does not have the
corresponding options set.
"""
- pass
+ autosummary_re = re.compile(r'^(\s*)\.\.\s+autosummary::\s*')
+ automodule_re = re.compile(r'^\s*\.\.\s+automodule::\s*([A-Za-z0-9_.]+)\s*$')
+ module_re = re.compile(r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$')
+ autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?')
+ recursive_arg_re = re.compile(r'^\s+:recursive:\s*$')
+ toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$')
+ template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$')
+
+ documented: list[AutosummaryEntry] = []
+
+ recursive = False
+ toctree: str | None = None
+ template = ''
+ current_module = module
+ in_autosummary = False
+ base_indent = ''
+
+ for line in lines:
+ if in_autosummary:
+ m = recursive_arg_re.match(line)
+ if m:
+ recursive = True
+ continue
+
+ m = toctree_arg_re.match(line)
+ if m:
+ toctree = m.group(1)
+ if filename:
+ toctree = os.path.join(os.path.dirname(filename), toctree)
+ continue
+
+ m = template_arg_re.match(line)
+ if m:
+ template = m.group(1).strip()
+ continue
+
+ if line.strip().startswith(':'):
+ continue # skip options
+
+ m = autosummary_item_re.match(line)
+ if m:
+ name = m.group(1).strip()
+ if name.startswith('~'):
+ name = name[1:]
+ if current_module and not name.startswith(current_module + '.'):
+ name = f'{current_module}.{name}'
+ documented.append(AutosummaryEntry(name, toctree, template, recursive))
+ continue
+
+ if not line.strip() or line.startswith(base_indent + ' '):
+ continue
+
+ in_autosummary = False
+
+ m = autosummary_re.match(line)
+ if m:
+ in_autosummary = True
+ base_indent = m.group(1)
+ recursive = False
+ toctree = None
+ template = ''
+ continue
+
+ m = automodule_re.search(line)
+ if m:
+ current_module = m.group(1).strip()
+ # recurse into the automodule docstring
+ documented.extend(
+ find_autosummary_in_docstring(current_module, filename=filename)
+ )
+ continue
+
+ m = module_re.match(line)
+ if m:
+ current_module = m.group(2)
+ continue
+
+ return documented
+
+
+def get_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ usage='%(prog)s [OPTIONS] <SOURCE_FILE>...',
+ epilog=__('For more information, visit <https://www.sphinx-doc.org/>.'),
+ description=__("""
+Generate ReStructuredText using autosummary directives.
+
+sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates
+the reStructuredText files from the autosummary directives contained in the
+given input files.
+
+The format of the autosummary directive is documented in the
+``sphinx.ext.autosummary`` Python module and can be read using::
+
+ pydoc sphinx.ext.autosummary
+"""),
+ )
+
+ parser.add_argument(
+ '--version',
+ action='version',
+ dest='show_version',
+ version='%%(prog)s %s' % __display_version__,
+ )
+
+ parser.add_argument(
+ 'source_file', nargs='+', help=__('source files to generate rST files for')
+ )
+
+ parser.add_argument(
+ '-o',
+ '--output-dir',
+ action='store',
+ dest='output_dir',
+ help=__('directory to place all output in'),
+ )
+ parser.add_argument(
+ '-s',
+ '--suffix',
+ action='store',
+ dest='suffix',
+ default='rst',
+ help=__('default suffix for files (default: %(default)s)'),
+ )
+ parser.add_argument(
+ '-t',
+ '--templates',
+ action='store',
+ dest='templates',
+ default=None,
+ help=__('custom template directory (default: %(default)s)'),
+ )
+ parser.add_argument(
+ '-i',
+ '--imported-members',
+ action='store_true',
+ dest='imported_members',
+ default=False,
+ help=__('document imported members (default: %(default)s)'),
+ )
+ parser.add_argument(
+ '-a',
+ '--respect-module-all',
+ action='store_true',
+ dest='respect_module_all',
+ default=False,
+ help=__(
+ 'document exactly the members in module __all__ attribute. '
+ '(default: %(default)s)'
+ ),
+ )
+ parser.add_argument(
+ '--remove-old',
+ action='store_true',
+ dest='remove_old',
+ default=False,
+ help=__(
+ 'Remove existing files in the output directory that were not generated'
+ ),
+ )
+
+ return parser
+
+
+def main(argv: Sequence[str] = (), /) -> None:
+ locale.setlocale(locale.LC_ALL, '')
+ sphinx.locale.init_console()
+
+ app = DummyApplication(sphinx.locale.get_translator())
+ logging.setup(app, sys.stdout, sys.stderr) # type: ignore[arg-type]
+ setup_documenters(app)
+ args = get_parser().parse_args(argv or sys.argv[1:])
+
+ if args.templates:
+ app.config.templates_path.append(path.abspath(args.templates))
+ app.config.autosummary_ignore_module_all = not args.respect_module_all
+
+ written_files = generate_autosummary_docs(
+ args.source_file,
+ args.output_dir,
+ '.' + args.suffix,
+ imported_members=args.imported_members,
+ app=app, # type: ignore[arg-type]
+ )
+
+ if args.remove_old:
+ for existing in Path(args.output_dir).glob(f'**/*.{args.suffix}'):
+ if existing not in written_files:
+ try:
+ existing.unlink()
+ except OSError as exc:
+ logger.warning(
+ __('Failed to remove %s: %s'),
+ existing,
+ exc.strerror,
+ type='autosummary',
+ )
if __name__ == '__main__':
diff --git a/sphinx/ext/coverage.py b/sphinx/ext/coverage.py
index 9e50f8dea..c075e9548 100644
--- a/sphinx/ext/coverage.py
+++ b/sphinx/ext/coverage.py
@@ -3,7 +3,9 @@
Mostly written by Josip Dzolonga for the Google Highly Open Participation
contest.
"""
+
from __future__ import annotations
+
import glob
import inspect
import pickle
@@ -13,21 +15,59 @@ import sys
from importlib import import_module
from os import path
from typing import IO, TYPE_CHECKING, Any, TextIO
+
import sphinx
from sphinx.builders import Builder
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.console import red
from sphinx.util.inspect import safe_getattr
+
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator, Sequence, Set
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
-def _load_modules(mod_name: str, ignored_module_exps: Iterable[re.Pattern[str]]
- ) ->Set[str]:
+# utility
+def write_header(f: IO[str], text: str, char: str = '-') -> None:
+ f.write(text + '\n')
+ f.write(char * len(text) + '\n\n')
+
+
+def compile_regex_list(name: str, exps: str) -> list[re.Pattern[str]]:
+ lst = []
+ for exp in exps:
+ try:
+ lst.append(re.compile(exp))
+ except Exception:
+ logger.warning(__('invalid regex %r in %s'), exp, name)
+ return lst
+
+
+def _write_table(table: list[list[str]]) -> Iterator[str]:
+ sizes = [max(len(x[column]) for x in table) + 1 for column in range(len(table[0]))]
+
+ yield _add_line(sizes, '-')
+ yield from _add_row(sizes, table[0], '=')
+
+ for row in table[1:]:
+ yield from _add_row(sizes, row, '-')
+
+
+def _add_line(sizes: list[int], separator: str) -> str:
+ return '+' + ''.join((separator * (size + 1)) + '+' for size in sizes)
+
+
+def _add_row(col_widths: list[int], columns: list[str], separator: str) -> Iterator[str]:
+ yield ''.join(f'| {column: <{col_widths[i]}}' for i, column in enumerate(columns)) + '|'
+ yield _add_line(col_widths, separator)
+
+
+def _load_modules(mod_name: str, ignored_module_exps: Iterable[re.Pattern[str]]) -> Set[str]:
"""Recursively load all submodules.
:param mod_name: The name of a module to load submodules for.
@@ -37,12 +77,36 @@ def _load_modules(mod_name: str, ignored_module_exps: Iterable[re.Pattern[str]]
:raises ImportError: If the module indicated by ``mod_name`` could not be
loaded.
"""
- pass
+ if any(exp.match(mod_name) for exp in ignored_module_exps):
+ return set()
+
+ # This can raise an exception, which must be handled by the caller.
+ mod = import_module(mod_name)
+ modules = {mod_name}
+ if mod.__spec__ is None:
+ return modules
+
+ search_locations = mod.__spec__.submodule_search_locations
+ for (_, sub_mod_name, sub_mod_ispkg) in pkgutil.iter_modules(search_locations):
+ if sub_mod_name == '__main__':
+ continue
+ if sub_mod_ispkg:
+ modules |= _load_modules(f'{mod_name}.{sub_mod_name}', ignored_module_exps)
+ else:
+ if any(exp.match(sub_mod_name) for exp in ignored_module_exps):
+ continue
+ modules.add(f'{mod_name}.{sub_mod_name}')
-def _determine_py_coverage_modules(coverage_modules: Sequence[str],
- seen_modules: Set[str], ignored_module_exps: Iterable[re.Pattern[str]],
- py_undoc: dict[str, dict[str, Any]]) ->list[str]:
+ return modules
+
+
+def _determine_py_coverage_modules(
+ coverage_modules: Sequence[str],
+ seen_modules: Set[str],
+ ignored_module_exps: Iterable[re.Pattern[str]],
+ py_undoc: dict[str, dict[str, Any]],
+) -> list[str]:
"""Return a sorted list of modules to check for coverage.
Figure out which of the two operating modes to use:
@@ -58,18 +122,375 @@ def _determine_py_coverage_modules(coverage_modules: Sequence[str],
modules that are documented will be noted. This will therefore identify both
missing modules and missing objects, but it requires manual configuration.
"""
- pass
+ if not coverage_modules:
+ return sorted(seen_modules)
+
+ modules: set[str] = set()
+ for mod_name in coverage_modules:
+ try:
+ modules |= _load_modules(mod_name, ignored_module_exps)
+ except ImportError as err:
+ # TODO(stephenfin): Define a subtype for all logs in this module
+ logger.warning(__('module %s could not be imported: %s'), mod_name, err)
+ py_undoc[mod_name] = {'error': err}
+ continue
+
+ # if there are additional modules then we warn but continue scanning
+ if additional_modules := seen_modules - modules:
+ logger.warning(
+ __('the following modules are documented but were not specified '
+ 'in coverage_modules: %s'),
+ ', '.join(additional_modules),
+ )
+
+ # likewise, if there are missing modules we warn but continue scanning
+ if missing_modules := modules - seen_modules:
+ logger.warning(
+ __('the following modules are specified in coverage_modules '
+ 'but were not documented'),
+ ', '.join(missing_modules),
+ )
+
+ return sorted(modules)
class CoverageBuilder(Builder):
"""
Evaluates coverage of code in the documentation.
"""
+
name = 'coverage'
- epilog = __(
- 'Testing of coverage in the sources finished, look at the results in %(outdir)s'
- + path.sep + 'python.txt.')
+ epilog = __('Testing of coverage in the sources finished, look at the '
+ 'results in %(outdir)s' + path.sep + 'python.txt.')
+
+ def init(self) -> None:
+ self.c_sourcefiles: list[str] = []
+ for pattern in self.config.coverage_c_path:
+ pattern = path.join(self.srcdir, pattern)
+ self.c_sourcefiles.extend(glob.glob(pattern))
+
+ self.c_regexes: list[tuple[str, re.Pattern[str]]] = []
+ for (name, exp) in self.config.coverage_c_regexes.items():
+ try:
+ self.c_regexes.append((name, re.compile(exp)))
+ except Exception:
+ logger.warning(__('invalid regex %r in coverage_c_regexes'), exp)
+
+ self.c_ignorexps: dict[str, list[re.Pattern[str]]] = {}
+ for (name, exps) in self.config.coverage_ignore_c_items.items():
+ self.c_ignorexps[name] = compile_regex_list('coverage_ignore_c_items',
+ exps)
+ self.mod_ignorexps = compile_regex_list('coverage_ignore_modules',
+ self.config.coverage_ignore_modules)
+ self.cls_ignorexps = compile_regex_list('coverage_ignore_classes',
+ self.config.coverage_ignore_classes)
+ self.fun_ignorexps = compile_regex_list('coverage_ignore_functions',
+ self.config.coverage_ignore_functions)
+ self.py_ignorexps = compile_regex_list('coverage_ignore_pyobjects',
+ self.config.coverage_ignore_pyobjects)
+
+ def get_outdated_docs(self) -> str:
+ return 'coverage overview'
+
+ def write(self, *ignored: Any) -> None:
+ self.py_undoc: dict[str, dict[str, Any]] = {}
+ self.py_undocumented: dict[str, Set[str]] = {}
+ self.py_documented: dict[str, Set[str]] = {}
+ self.build_py_coverage()
+ self.write_py_coverage()
- def _write_py_statistics(self, op: TextIO) ->None:
+ self.c_undoc: dict[str, Set[tuple[str, str]]] = {}
+ self.build_c_coverage()
+ self.write_c_coverage()
+
+ def build_c_coverage(self) -> None:
+ c_objects = {}
+ for obj in self.env.domains['c'].get_objects():
+ c_objects[obj[2]] = obj[1]
+ for filename in self.c_sourcefiles:
+ undoc: set[tuple[str, str]] = set()
+ with open(filename, encoding="utf-8") as f:
+ for line in f:
+ for key, regex in self.c_regexes:
+ match = regex.match(line)
+ if match:
+ name = match.groups()[0]
+ if key not in c_objects:
+ undoc.add((key, name))
+ continue
+
+ if name not in c_objects[key]:
+ for exp in self.c_ignorexps.get(key, []):
+ if exp.match(name):
+ break
+ else:
+ undoc.add((key, name))
+ continue
+ if undoc:
+ self.c_undoc[filename] = undoc
+
+ def write_c_coverage(self) -> None:
+ output_file = path.join(self.outdir, 'c.txt')
+ with open(output_file, 'w', encoding="utf-8") as op:
+ if self.config.coverage_write_headline:
+ write_header(op, 'Undocumented C API elements', '=')
+ op.write('\n')
+
+ for filename, undoc in self.c_undoc.items():
+ write_header(op, filename)
+ for typ, name in sorted(undoc):
+ op.write(' * %-50s [%9s]\n' % (name, typ))
+ if self.config.coverage_show_missing_items:
+ if self.app.quiet:
+ logger.warning(__('undocumented c api: %s [%s] in file %s'),
+ name, typ, filename)
+ else:
+ logger.info(red('undocumented ') + 'c ' + 'api ' +
+ '%-30s' % (name + " [%9s]" % typ) +
+ red(' - in file ') + filename)
+ op.write('\n')
+
+ def ignore_pyobj(self, full_name: str) -> bool:
+ return any(
+ exp.search(full_name)
+ for exp in self.py_ignorexps
+ )
+
+ def build_py_coverage(self) -> None:
+ seen_objects = frozenset(self.env.domaindata['py']['objects'])
+ seen_modules = frozenset(self.env.domaindata['py']['modules'])
+
+ skip_undoc = self.config.coverage_skip_undoc_in_source
+
+ modules = _determine_py_coverage_modules(
+ self.config.coverage_modules, seen_modules, self.mod_ignorexps, self.py_undoc,
+ )
+ for mod_name in modules:
+ ignore = False
+ for exp in self.mod_ignorexps:
+ if exp.match(mod_name):
+ ignore = True
+ break
+ if ignore or self.ignore_pyobj(mod_name):
+ continue
+
+ try:
+ mod = import_module(mod_name)
+ except ImportError as err:
+ logger.warning(__('module %s could not be imported: %s'), mod_name, err)
+ self.py_undoc[mod_name] = {'error': err}
+ continue
+
+ documented_objects: set[str] = set()
+ undocumented_objects: set[str] = set()
+
+ funcs = []
+ classes: dict[str, list[str]] = {}
+
+ for name, obj in inspect.getmembers(mod):
+ # diverse module attributes are ignored:
+ if name[0] == '_':
+ # begins in an underscore
+ continue
+ if not hasattr(obj, '__module__'):
+ # cannot be attributed to a module
+ continue
+ if obj.__module__ != mod_name:
+ # is not defined in this module
+ continue
+
+ full_name = f'{mod_name}.{name}'
+ if self.ignore_pyobj(full_name):
+ continue
+
+ if inspect.isfunction(obj):
+ if full_name not in seen_objects:
+ for exp in self.fun_ignorexps:
+ if exp.match(name):
+ break
+ else:
+ if skip_undoc and not obj.__doc__:
+ continue
+ funcs.append(name)
+ undocumented_objects.add(full_name)
+ else:
+ documented_objects.add(full_name)
+ elif inspect.isclass(obj):
+ for exp in self.cls_ignorexps:
+ if exp.match(name):
+ break
+ else:
+ if full_name not in seen_objects:
+ if skip_undoc and not obj.__doc__:
+ continue
+ # not documented at all
+ classes[name] = []
+ continue
+
+ attrs: list[str] = []
+
+ for attr_name in dir(obj):
+ if attr_name not in obj.__dict__:
+ continue
+ try:
+ attr = safe_getattr(obj, attr_name)
+ except AttributeError:
+ continue
+ if not (inspect.ismethod(attr) or
+ inspect.isfunction(attr)):
+ continue
+ if attr_name[0] == '_':
+ # starts with an underscore, ignore it
+ continue
+ if skip_undoc and not attr.__doc__:
+ # skip methods without docstring if wished
+ continue
+ full_attr_name = f'{full_name}.{attr_name}'
+ if self.ignore_pyobj(full_attr_name):
+ continue
+ if full_attr_name not in seen_objects:
+ attrs.append(attr_name)
+ undocumented_objects.add(full_attr_name)
+ else:
+ documented_objects.add(full_attr_name)
+
+ if attrs:
+ # some attributes are undocumented
+ classes[name] = attrs
+
+ self.py_undoc[mod_name] = {'funcs': funcs, 'classes': classes}
+ self.py_undocumented[mod_name] = undocumented_objects
+ self.py_documented[mod_name] = documented_objects
+
+ def _write_py_statistics(self, op: TextIO) -> None:
"""Outputs the table of ``op``."""
- pass
+ all_modules = frozenset(self.py_documented.keys() | self.py_undocumented.keys())
+ all_objects: Set[str] = set()
+ all_documented_objects: Set[str] = set()
+ for module in all_modules:
+ all_objects |= self.py_documented[module] | self.py_undocumented[module]
+ all_documented_objects |= self.py_documented[module]
+
+ # prepare tabular
+ table = [['Module', 'Coverage', 'Undocumented']]
+ for module in sorted(all_modules):
+ module_objects = self.py_documented[module] | self.py_undocumented[module]
+ if len(module_objects):
+ value = 100.0 * len(self.py_documented[module]) / len(module_objects)
+ else:
+ value = 100.0
+
+ table.append([module, '%.2f%%' % value, '%d' % len(self.py_undocumented[module])])
+
+ if all_objects:
+ table.append([
+ 'TOTAL',
+ f'{100 * len(all_documented_objects) / len(all_objects):.2f}%',
+ f'{len(all_objects) - len(all_documented_objects)}',
+ ])
+ else:
+ table.append(['TOTAL', '100', '0'])
+
+ for line in _write_table(table):
+ op.write(f'{line}\n')
+
+ def write_py_coverage(self) -> None:
+ output_file = path.join(self.outdir, 'python.txt')
+ failed = []
+ with open(output_file, 'w', encoding="utf-8") as op:
+ if self.config.coverage_write_headline:
+ write_header(op, 'Undocumented Python objects', '=')
+
+ if self.config.coverage_statistics_to_stdout:
+ self._write_py_statistics(sys.stdout)
+
+ if self.config.coverage_statistics_to_report:
+ write_header(op, 'Statistics')
+ self._write_py_statistics(op)
+ op.write('\n')
+
+ keys = sorted(self.py_undoc.keys())
+ for name in keys:
+ undoc = self.py_undoc[name]
+ if 'error' in undoc:
+ failed.append((name, undoc['error']))
+ else:
+ if not undoc['classes'] and not undoc['funcs']:
+ continue
+
+ write_header(op, name)
+ if undoc['funcs']:
+ op.write('Functions:\n')
+ op.writelines(' * %s\n' % x for x in undoc['funcs'])
+ if self.config.coverage_show_missing_items:
+ if self.app.quiet:
+ for func in undoc['funcs']:
+ logger.warning(
+ __('undocumented python function: %s :: %s'),
+ name, func)
+ else:
+ for func in undoc['funcs']:
+ logger.info(red('undocumented ') + 'py ' + 'function ' +
+ '%-30s' % func + red(' - in module ') + name)
+ op.write('\n')
+ if undoc['classes']:
+ op.write('Classes:\n')
+ for class_name, methods in sorted(
+ undoc['classes'].items()):
+ if not methods:
+ op.write(' * %s\n' % class_name)
+ if self.config.coverage_show_missing_items:
+ if self.app.quiet:
+ logger.warning(
+ __('undocumented python class: %s :: %s'),
+ name, class_name)
+ else:
+ logger.info(red('undocumented ') + 'py ' +
+ 'class ' + '%-30s' % class_name +
+ red(' - in module ') + name)
+ else:
+ op.write(' * %s -- missing methods:\n\n' % class_name)
+ op.writelines(' - %s\n' % x for x in methods)
+ if self.config.coverage_show_missing_items:
+ if self.app.quiet:
+ for meth in methods:
+ logger.warning(
+ __('undocumented python method:'
+ ' %s :: %s :: %s'),
+ name, class_name, meth)
+ else:
+ for meth in methods:
+ logger.info(red('undocumented ') + 'py ' +
+ 'method ' + '%-30s' %
+ (class_name + '.' + meth) +
+ red(' - in module ') + name)
+ op.write('\n')
+
+ if failed:
+ write_header(op, 'Modules that failed to import')
+ op.writelines(' * %s -- %s\n' % x for x in failed)
+
+ def finish(self) -> None:
+ # dump the coverage data to a pickle file too
+ picklepath = path.join(self.outdir, 'undoc.pickle')
+ with open(picklepath, 'wb') as dumpfile:
+ pickle.dump((self.py_undoc, self.c_undoc,
+ self.py_undocumented, self.py_documented), dumpfile)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_builder(CoverageBuilder)
+ app.add_config_value('coverage_modules', (), '', types={tuple, list})
+ app.add_config_value('coverage_ignore_modules', [], '')
+ app.add_config_value('coverage_ignore_functions', [], '')
+ app.add_config_value('coverage_ignore_classes', [], '')
+ app.add_config_value('coverage_ignore_pyobjects', [], '')
+ app.add_config_value('coverage_c_path', [], '')
+ app.add_config_value('coverage_c_regexes', {}, '')
+ app.add_config_value('coverage_ignore_c_items', {}, '')
+ app.add_config_value('coverage_write_headline', True, '')
+ app.add_config_value('coverage_statistics_to_report', True, '', bool)
+ app.add_config_value('coverage_statistics_to_stdout', True, '', bool)
+ app.add_config_value('coverage_skip_undoc_in_source', False, '')
+ app.add_config_value('coverage_show_missing_items', False, '')
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py
index 46e4ad53d..ba451208a 100644
--- a/sphinx/ext/doctest.py
+++ b/sphinx/ext/doctest.py
@@ -2,7 +2,9 @@
The extension automatically execute code snippets and checks their results.
"""
+
from __future__ import annotations
+
import doctest
import re
import sys
@@ -10,10 +12,12 @@ import time
from io import StringIO
from os import path
from typing import TYPE_CHECKING, Any, ClassVar
+
from docutils import nodes
from docutils.parsers.rst import directives
from packaging.specifiers import InvalidSpecifier, SpecifierSet
from packaging.version import Version
+
import sphinx
from sphinx.builders import Builder
from sphinx.locale import __
@@ -21,17 +25,23 @@ from sphinx.util import logging
from sphinx.util.console import bold
from sphinx.util.docutils import SphinxDirective
from sphinx.util.osutil import relpath
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Sequence
+
from docutils.nodes import Element, Node, TextElement
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata, OptionSpec
+
+
logger = logging.getLogger(__name__)
-blankline_re = re.compile('^\\s*<BLANKLINE>', re.MULTILINE)
-doctestopt_re = re.compile('#\\s*doctest:.+$', re.MULTILINE)
+
+blankline_re = re.compile(r'^\s*<BLANKLINE>', re.MULTILINE)
+doctestopt_re = re.compile(r'#\s*doctest:.+$', re.MULTILINE)
-def is_allowed_version(spec: str, version: str) ->bool:
+def is_allowed_version(spec: str, version: str) -> bool:
"""Check `spec` satisfies `version` or not.
This obeys PEP-440 specifiers:
@@ -46,107 +56,525 @@ def is_allowed_version(spec: str, version: str) ->bool:
>>> is_allowed_version('>3.2, <4.0', '3.3')
True
"""
- pass
+ return Version(version) in SpecifierSet(spec)
+# set up the necessary directives
+
class TestDirective(SphinxDirective):
"""
Base class for doctest-related directives.
"""
+
has_content = True
required_arguments = 0
optional_arguments = 1
final_argument_whitespace = True
+ def run(self) -> list[Node]:
+ # use ordinary docutils nodes for test code: they get special attributes
+ # so that our builder recognizes them, and the other builders are happy.
+ code = '\n'.join(self.content)
+ test = None
+ if self.name == 'doctest':
+ if '<BLANKLINE>' in code:
+ # convert <BLANKLINE>s to ordinary blank lines for presentation
+ test = code
+ code = blankline_re.sub('', code)
+ if doctestopt_re.search(code) and 'no-trim-doctest-flags' not in self.options:
+ if not test:
+ test = code
+ code = doctestopt_re.sub('', code)
+ nodetype: type[TextElement] = nodes.literal_block
+ if self.name in ('testsetup', 'testcleanup') or 'hide' in self.options:
+ nodetype = nodes.comment
+ if self.arguments:
+ groups = [x.strip() for x in self.arguments[0].split(',')]
+ else:
+ groups = ['default']
+ node = nodetype(code, code, testnodetype=self.name, groups=groups)
+ self.set_source_info(node)
+ if test is not None:
+ # only save if it differs from code
+ node['test'] = test
+ if self.name == 'doctest':
+ node['language'] = 'pycon'
+ elif self.name == 'testcode':
+ node['language'] = 'python'
+ elif self.name == 'testoutput':
+ # don't try to highlight output
+ node['language'] = 'none'
+ node['options'] = {}
+ if self.name in ('doctest', 'testoutput') and 'options' in self.options:
+ # parse doctest-like output comparison flags
+ option_strings = self.options['options'].replace(',', ' ').split()
+ for option in option_strings:
+ prefix, option_name = option[0], option[1:]
+ if prefix not in '+-':
+ self.state.document.reporter.warning(
+ __("missing '+' or '-' in '%s' option.") % option,
+ line=self.lineno)
+ continue
+ if option_name not in doctest.OPTIONFLAGS_BY_NAME:
+ self.state.document.reporter.warning(
+ __("'%s' is not a valid option.") % option_name,
+ line=self.lineno)
+ continue
+ flag = doctest.OPTIONFLAGS_BY_NAME[option[1:]]
+ node['options'][flag] = (option[0] == '+')
+ if self.name == 'doctest' and 'pyversion' in self.options:
+ try:
+ spec = self.options['pyversion']
+ python_version = '.'.join(map(str, sys.version_info[:3]))
+ if not is_allowed_version(spec, python_version):
+ flag = doctest.OPTIONFLAGS_BY_NAME['SKIP']
+ node['options'][flag] = True # Skip the test
+ except InvalidSpecifier:
+ self.state.document.reporter.warning(
+ __("'%s' is not a valid pyversion option") % spec,
+ line=self.lineno)
+ if 'skipif' in self.options:
+ node['skipif'] = self.options['skipif']
+ if 'trim-doctest-flags' in self.options:
+ node['trim_flags'] = True
+ elif 'no-trim-doctest-flags' in self.options:
+ node['trim_flags'] = False
+ return [node]
+
class TestsetupDirective(TestDirective):
- option_spec: ClassVar[OptionSpec] = {'skipif': directives.
- unchanged_required}
+ option_spec: ClassVar[OptionSpec] = {
+ 'skipif': directives.unchanged_required,
+ }
class TestcleanupDirective(TestDirective):
- option_spec: ClassVar[OptionSpec] = {'skipif': directives.
- unchanged_required}
+ option_spec: ClassVar[OptionSpec] = {
+ 'skipif': directives.unchanged_required,
+ }
class DoctestDirective(TestDirective):
- option_spec: ClassVar[OptionSpec] = {'hide': directives.flag,
- 'no-trim-doctest-flags': directives.flag, 'options': directives.
- unchanged, 'pyversion': directives.unchanged_required, 'skipif':
- directives.unchanged_required, 'trim-doctest-flags': directives.flag}
+ option_spec: ClassVar[OptionSpec] = {
+ 'hide': directives.flag,
+ 'no-trim-doctest-flags': directives.flag,
+ 'options': directives.unchanged,
+ 'pyversion': directives.unchanged_required,
+ 'skipif': directives.unchanged_required,
+ 'trim-doctest-flags': directives.flag,
+ }
class TestcodeDirective(TestDirective):
- option_spec: ClassVar[OptionSpec] = {'hide': directives.flag,
- 'no-trim-doctest-flags': directives.flag, 'pyversion': directives.
- unchanged_required, 'skipif': directives.unchanged_required,
- 'trim-doctest-flags': directives.flag}
+ option_spec: ClassVar[OptionSpec] = {
+ 'hide': directives.flag,
+ 'no-trim-doctest-flags': directives.flag,
+ 'pyversion': directives.unchanged_required,
+ 'skipif': directives.unchanged_required,
+ 'trim-doctest-flags': directives.flag,
+ }
class TestoutputDirective(TestDirective):
- option_spec: ClassVar[OptionSpec] = {'hide': directives.flag,
- 'no-trim-doctest-flags': directives.flag, 'options': directives.
- unchanged, 'pyversion': directives.unchanged_required, 'skipif':
- directives.unchanged_required, 'trim-doctest-flags': directives.flag}
+ option_spec: ClassVar[OptionSpec] = {
+ 'hide': directives.flag,
+ 'no-trim-doctest-flags': directives.flag,
+ 'options': directives.unchanged,
+ 'pyversion': directives.unchanged_required,
+ 'skipif': directives.unchanged_required,
+ 'trim-doctest-flags': directives.flag,
+ }
parser = doctest.DocTestParser()
-class TestGroup:
+# helper classes
- def __init__(self, name: str) ->None:
+class TestGroup:
+ def __init__(self, name: str) -> None:
self.name = name
self.setup: list[TestCode] = []
self.tests: list[list[TestCode] | tuple[TestCode, None]] = []
self.cleanup: list[TestCode] = []
- def __repr__(self) ->str:
- return (
- f'TestGroup(name={self.name!r}, setup={self.setup!r}, cleanup={self.cleanup!r}, tests={self.tests!r})'
- )
+ def add_code(self, code: TestCode, prepend: bool = False) -> None:
+ if code.type == 'testsetup':
+ if prepend:
+ self.setup.insert(0, code)
+ else:
+ self.setup.append(code)
+ elif code.type == 'testcleanup':
+ self.cleanup.append(code)
+ elif code.type == 'doctest':
+ self.tests.append([code])
+ elif code.type == 'testcode':
+ # "testoutput" may replace the second element
+ self.tests.append((code, None))
+ elif code.type == 'testoutput':
+ if self.tests:
+ latest_test = self.tests[-1]
+ if len(latest_test) == 2:
+ self.tests[-1] = [latest_test[0], code]
+ else:
+ raise RuntimeError(__('invalid TestCode type'))
+
+ def __repr__(self) -> str:
+ return (f'TestGroup(name={self.name!r}, setup={self.setup!r}, '
+ f'cleanup={self.cleanup!r}, tests={self.tests!r})')
class TestCode:
-
- def __init__(self, code: str, type: str, filename: str, lineno: int,
- options: (dict | None)=None) ->None:
+ def __init__(self, code: str, type: str, filename: str,
+ lineno: int, options: dict | None = None) -> None:
self.code = code
self.type = type
self.filename = filename
self.lineno = lineno
self.options = options or {}
- def __repr__(self) ->str:
- return (
- f'TestCode({self.code!r}, {self.type!r}, filename={self.filename!r}, lineno={self.lineno!r}, options={self.options!r})'
- )
+ def __repr__(self) -> str:
+ return (f'TestCode({self.code!r}, {self.type!r}, filename={self.filename!r}, '
+ f'lineno={self.lineno!r}, options={self.options!r})')
class SphinxDocTestRunner(doctest.DocTestRunner):
- pass
-
+ def summarize(self, out: Callable, verbose: bool | None = None, # type: ignore[override]
+ ) -> tuple[int, int]:
+ string_io = StringIO()
+ old_stdout = sys.stdout
+ sys.stdout = string_io
+ try:
+ res = super().summarize(verbose)
+ finally:
+ sys.stdout = old_stdout
+ out(string_io.getvalue())
+ return res
+
+ def _DocTestRunner__patched_linecache_getlines(self, filename: str,
+ module_globals: Any = None) -> Any:
+ # this is overridden from DocTestRunner adding the try-except below
+ m = self._DocTestRunner__LINECACHE_FILENAME_RE.match( # type: ignore[attr-defined]
+ filename)
+ if m and m.group('name') == self.test.name:
+ try:
+ example = self.test.examples[int(m.group('examplenum'))]
+ # because we compile multiple doctest blocks with the same name
+ # (viz. the group name) this might, for outer stack frames in a
+ # traceback, get the wrong test which might not have enough examples
+ except IndexError:
+ pass
+ else:
+ return example.source.splitlines(True)
+ return self.save_linecache_getlines( # type: ignore[attr-defined]
+ filename, module_globals)
+
+
+# the new builder -- use sphinx-build.py -b doctest to run
class DocTestBuilder(Builder):
"""
Runs test snippets in the documentation.
"""
- name = 'doctest'
- epilog = __(
- 'Testing of doctests in the sources finished, look at the results in %(outdir)s/output.txt.'
- )
- def __del__(self) ->None:
+ name = 'doctest'
+ epilog = __('Testing of doctests in the sources finished, look at the '
+ 'results in %(outdir)s/output.txt.')
+
+ def init(self) -> None:
+ # default options
+ self.opt = self.config.doctest_default_flags
+
+ # HACK HACK HACK
+ # doctest compiles its snippets with type 'single'. That is nice
+ # for doctest examples but unusable for multi-statement code such
+ # as setup code -- to be able to use doctest error reporting with
+ # that code nevertheless, we monkey-patch the "compile" it uses.
+ doctest.compile = self.compile # type: ignore[attr-defined]
+
+ sys.path[0:0] = self.config.doctest_path
+
+ self.type = 'single'
+
+ self.total_failures = 0
+ self.total_tries = 0
+ self.setup_failures = 0
+ self.setup_tries = 0
+ self.cleanup_failures = 0
+ self.cleanup_tries = 0
+
+ date = time.strftime('%Y-%m-%d %H:%M:%S')
+
+ outpath = self.outdir.joinpath('output.txt')
+ self.outfile = outpath.open('w', encoding='utf-8') # NoQA: SIM115
+ self.outfile.write(('Results of doctest builder run on %s\n'
+ '==================================%s\n') %
+ (date, '=' * len(date)))
+
+ def __del__(self) -> None:
+ # free resources upon destruction (the file handler might not be
+ # closed if the builder is never used)
if hasattr(self, 'outfile'):
self.outfile.close()
- def get_filename_for_node(self, node: Node, docname: str) ->str:
+ def _out(self, text: str) -> None:
+ logger.info(text, nonl=True)
+ self.outfile.write(text)
+
+ def _warn_out(self, text: str) -> None:
+ if self.app.quiet:
+ logger.warning(text)
+ else:
+ logger.info(text, nonl=True)
+ self.outfile.write(text)
+
+ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+ return ''
+
+ def get_outdated_docs(self) -> set[str]:
+ return self.env.found_docs
+
+ def finish(self) -> None:
+ # write executive summary
+ def s(v: int) -> str:
+ return 's' if v != 1 else ''
+ repl = (self.total_tries, s(self.total_tries),
+ self.total_failures, s(self.total_failures),
+ self.setup_failures, s(self.setup_failures),
+ self.cleanup_failures, s(self.cleanup_failures))
+ self._out('''
+Doctest summary
+===============
+%5d test%s
+%5d failure%s in tests
+%5d failure%s in setup code
+%5d failure%s in cleanup code
+''' % repl)
+ self.outfile.close()
+
+ if self.total_failures or self.setup_failures or self.cleanup_failures:
+ self.app.statuscode = 1
+
+ def write(self, build_docnames: Iterable[str] | None, updated_docnames: Sequence[str],
+ method: str = 'update') -> None:
+ if build_docnames is None:
+ build_docnames = sorted(self.env.all_docs)
+
+ logger.info(bold('running tests...'))
+ for docname in build_docnames:
+ # no need to resolve the doctree
+ doctree = self.env.get_doctree(docname)
+ self.test_doc(docname, doctree)
+
+ def get_filename_for_node(self, node: Node, docname: str) -> str:
"""Try to get the file which actually contains the doctest, not the
filename of the document it's included in.
"""
- pass
+ try:
+ filename = relpath(node.source, self.env.srcdir).rsplit(':docstring of ', maxsplit=1)[0] # type: ignore[arg-type] # noqa: E501
+ except Exception:
+ filename = str(self.env.doc2path(docname, False))
+ return filename
@staticmethod
- def get_line_number(node: Node) ->(int | None):
+ def get_line_number(node: Node) -> int | None:
"""Get the real line number or admit we don't know."""
- pass
+ # TODO: Work out how to store or calculate real (file-relative)
+ # line numbers for doctest blocks in docstrings.
+ if ':docstring of ' in path.basename(node.source or ''):
+ # The line number is given relative to the stripped docstring,
+ # not the file. This is correct where it is set, in
+ # `docutils.nodes.Node.setup_child`, but Sphinx should report
+ # relative to the file, not the docstring.
+ return None
+ if node.line is not None:
+ # TODO: find the root cause of this off by one error.
+ return node.line - 1
+ return None
+
+ def skipped(self, node: Element) -> bool:
+ if 'skipif' not in node:
+ return False
+ else:
+ condition = node['skipif']
+ context: dict[str, Any] = {}
+ if self.config.doctest_global_setup:
+ exec(self.config.doctest_global_setup, context) # NoQA: S102
+ should_skip = eval(condition, context) # NoQA: S307
+ if self.config.doctest_global_cleanup:
+ exec(self.config.doctest_global_cleanup, context) # NoQA: S102
+ return should_skip
+
+ def test_doc(self, docname: str, doctree: Node) -> None:
+ groups: dict[str, TestGroup] = {}
+ add_to_all_groups = []
+ self.setup_runner = SphinxDocTestRunner(verbose=False,
+ optionflags=self.opt)
+ self.test_runner = SphinxDocTestRunner(verbose=False,
+ optionflags=self.opt)
+ self.cleanup_runner = SphinxDocTestRunner(verbose=False,
+ optionflags=self.opt)
+
+ self.test_runner._fakeout = self.setup_runner._fakeout # type: ignore[attr-defined]
+ self.cleanup_runner._fakeout = self.setup_runner._fakeout # type: ignore[attr-defined]
+
+ if self.config.doctest_test_doctest_blocks:
+ def condition(node: Node) -> bool:
+ return (isinstance(node, nodes.literal_block | nodes.comment) and
+ 'testnodetype' in node) or \
+ isinstance(node, nodes.doctest_block)
+ else:
+ def condition(node: Node) -> bool:
+ return isinstance(node, nodes.literal_block | nodes.comment) \
+ and 'testnodetype' in node
+ for node in doctree.findall(condition):
+ if self.skipped(node): # type: ignore[arg-type]
+ continue
+
+ source = node['test'] if 'test' in node else node.astext() # type: ignore[index, operator]
+ filename = self.get_filename_for_node(node, docname)
+ line_number = self.get_line_number(node)
+ if not source:
+ logger.warning(__('no code/output in %s block at %s:%s'),
+ node.get('testnodetype', 'doctest'), # type: ignore[attr-defined]
+ filename, line_number)
+ code = TestCode(source, type=node.get('testnodetype', 'doctest'), # type: ignore[attr-defined]
+ filename=filename, lineno=line_number, # type: ignore[arg-type]
+ options=node.get('options')) # type: ignore[attr-defined]
+ node_groups = node.get('groups', ['default']) # type: ignore[attr-defined]
+ if '*' in node_groups:
+ add_to_all_groups.append(code)
+ continue
+ for groupname in node_groups:
+ if groupname not in groups:
+ groups[groupname] = TestGroup(groupname)
+ groups[groupname].add_code(code)
+ for code in add_to_all_groups:
+ for group in groups.values():
+ group.add_code(code)
+ if self.config.doctest_global_setup:
+ code = TestCode(self.config.doctest_global_setup,
+ 'testsetup', filename='<global_setup>', lineno=0)
+ for group in groups.values():
+ group.add_code(code, prepend=True)
+ if self.config.doctest_global_cleanup:
+ code = TestCode(self.config.doctest_global_cleanup,
+ 'testcleanup', filename='<global_cleanup>', lineno=0)
+ for group in groups.values():
+ group.add_code(code)
+ if not groups:
+ return
+
+ show_successes = self.config.doctest_show_successes
+ if show_successes:
+ self._out('\n'
+ f'Document: {docname}\n'
+ f'----------{"-" * len(docname)}\n')
+ for group in groups.values():
+ self.test_group(group)
+ # Separately count results from setup code
+ res_f, res_t = self.setup_runner.summarize(self._out, verbose=False)
+ self.setup_failures += res_f
+ self.setup_tries += res_t
+ if self.test_runner.tries:
+ res_f, res_t = self.test_runner.summarize(
+ self._out, verbose=show_successes)
+ self.total_failures += res_f
+ self.total_tries += res_t
+ if self.cleanup_runner.tries:
+ res_f, res_t = self.cleanup_runner.summarize(
+ self._out, verbose=show_successes)
+ self.cleanup_failures += res_f
+ self.cleanup_tries += res_t
+
+ def compile(self, code: str, name: str, type: str, flags: Any, dont_inherit: bool) -> Any:
+ return compile(code, name, self.type, flags, dont_inherit)
+
+ def test_group(self, group: TestGroup) -> None:
+ ns: dict = {}
+
+ def run_setup_cleanup(runner: Any, testcodes: list[TestCode], what: Any) -> bool:
+ examples = []
+ for testcode in testcodes:
+ example = doctest.Example(testcode.code, '', lineno=testcode.lineno)
+ examples.append(example)
+ if not examples:
+ return True
+ # simulate a doctest with the code
+ sim_doctest = doctest.DocTest(examples, {},
+ f'{group.name} ({what} code)',
+ testcodes[0].filename, 0, None)
+ sim_doctest.globs = ns
+ old_f = runner.failures
+ self.type = 'exec' # the snippet may contain multiple statements
+ runner.run(sim_doctest, out=self._warn_out, clear_globs=False)
+ return runner.failures <= old_f
+
+ # run the setup code
+ if not run_setup_cleanup(self.setup_runner, group.setup, 'setup'):
+ # if setup failed, don't run the group
+ return
+
+ # run the tests
+ for code in group.tests:
+ if len(code) == 1:
+ # ordinary doctests (code/output interleaved)
+ try:
+ test = parser.get_doctest(code[0].code, {}, group.name,
+ code[0].filename, code[0].lineno)
+ except Exception:
+ logger.warning(__('ignoring invalid doctest code: %r'), code[0].code,
+ location=(code[0].filename, code[0].lineno))
+ continue
+ if not test.examples:
+ continue
+ for example in test.examples:
+ # apply directive's comparison options
+ new_opt = code[0].options.copy()
+ new_opt.update(example.options)
+ example.options = new_opt
+ self.type = 'single' # as for ordinary doctests
+ else:
+ # testcode and output separate
+ output = code[1].code if code[1] else ''
+ options = code[1].options if code[1] else {}
+ # disable <BLANKLINE> processing as it is not needed
+ options[doctest.DONT_ACCEPT_BLANKLINE] = True
+ # find out if we're testing an exception
+ m = parser._EXCEPTION_RE.match(output) # type: ignore[attr-defined]
+ if m:
+ exc_msg = m.group('msg')
+ else:
+ exc_msg = None
+ example = doctest.Example(code[0].code, output, exc_msg=exc_msg,
+ lineno=code[0].lineno, options=options)
+ test = doctest.DocTest([example], {}, group.name,
+ code[0].filename, code[0].lineno, None)
+ self.type = 'exec' # multiple statements again
+ # DocTest.__init__ copies the globs namespace, which we don't want
+ test.globs = ns
+ # also don't clear the globs namespace after running the doctest
+ self.test_runner.run(test, out=self._warn_out, clear_globs=False)
+
+ # run the cleanup
+ run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup')
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_directive('testsetup', TestsetupDirective)
+ app.add_directive('testcleanup', TestcleanupDirective)
+ app.add_directive('doctest', DoctestDirective)
+ app.add_directive('testcode', TestcodeDirective)
+ app.add_directive('testoutput', TestoutputDirective)
+ app.add_builder(DocTestBuilder)
+ # this config value adds to sys.path
+ app.add_config_value('doctest_show_successes', True, '', bool)
+ app.add_config_value('doctest_path', [], '')
+ app.add_config_value('doctest_test_doctest_blocks', 'default', '')
+ app.add_config_value('doctest_global_setup', '', '')
+ app.add_config_value('doctest_global_cleanup', '', '')
+ app.add_config_value(
+ 'doctest_default_flags',
+ doctest.DONT_ACCEPT_TRUE_FOR_1 | doctest.ELLIPSIS | doctest.IGNORE_EXCEPTION_DETAIL,
+ '')
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/duration.py b/sphinx/ext/duration.py
index 9080b706a..1053856e0 100644
--- a/sphinx/ext/duration.py
+++ b/sphinx/ext/duration.py
@@ -1,47 +1,100 @@
"""Measure document reading durations."""
+
from __future__ import annotations
+
import time
from itertools import islice
from operator import itemgetter
from typing import TYPE_CHECKING, cast
+
import sphinx
from sphinx.domains import Domain
from sphinx.locale import __
from sphinx.util import logging
+
if TYPE_CHECKING:
from typing import TypedDict
+
from docutils import nodes
- from sphinx.application import Sphinx
+ from sphinx.application import Sphinx
class _DurationDomainData(TypedDict):
reading_durations: dict[str, float]
+
logger = logging.getLogger(__name__)
class DurationDomain(Domain):
"""A domain for durations of Sphinx processing."""
+
name = 'duration'
+ @property
+ def reading_durations(self) -> dict[str, float]:
+ return self.data.setdefault('reading_durations', {})
-def on_builder_inited(app: Sphinx) ->None:
+ def note_reading_duration(self, duration: float) -> None:
+ self.reading_durations[self.env.docname] = duration
+
+ def clear(self) -> None:
+ self.reading_durations.clear()
+
+ def clear_doc(self, docname: str) -> None:
+ self.reading_durations.pop(docname, None)
+
+ def merge_domaindata(self, docnames: list[str], otherdata: _DurationDomainData) -> None: # type: ignore[override]
+ other_reading_durations = otherdata.get('reading_durations', {})
+ docnames_set = frozenset(docnames)
+ for docname, duration in other_reading_durations.items():
+ if docname in docnames_set:
+ self.reading_durations[docname] = duration
+
+
+def on_builder_inited(app: Sphinx) -> None:
"""Initialize DurationDomain on bootstrap.
This clears the results of the last build.
"""
- pass
+ domain = cast(DurationDomain, app.env.get_domain('duration'))
+ domain.clear()
-def on_source_read(app: Sphinx, docname: str, content: list[str]) ->None:
+def on_source_read(app: Sphinx, docname: str, content: list[str]) -> None:
"""Start to measure reading duration."""
- pass
+ app.env.temp_data['started_at'] = time.monotonic()
-def on_doctree_read(app: Sphinx, doctree: nodes.document) ->None:
+def on_doctree_read(app: Sphinx, doctree: nodes.document) -> None:
"""Record a reading duration."""
- pass
+ started_at = app.env.temp_data['started_at']
+ duration = time.monotonic() - started_at
+ domain = cast(DurationDomain, app.env.get_domain('duration'))
+ domain.note_reading_duration(duration)
-def on_build_finished(app: Sphinx, error: Exception) ->None:
+def on_build_finished(app: Sphinx, error: Exception) -> None:
"""Display duration ranking on the current build."""
- pass
+ domain = cast(DurationDomain, app.env.get_domain('duration'))
+ if not domain.reading_durations:
+ return
+ durations = sorted(domain.reading_durations.items(), key=itemgetter(1), reverse=True)
+
+ logger.info('')
+ logger.info(__('====================== slowest reading durations ======================='))
+ for docname, d in islice(durations, 5):
+ logger.info(f'{d:.3f} {docname}') # NoQA: G004
+
+
+def setup(app: Sphinx) -> dict[str, bool | str]:
+ app.add_domain(DurationDomain)
+ app.connect('builder-inited', on_builder_inited)
+ app.connect('source-read', on_source_read)
+ app.connect('doctree-read', on_doctree_read)
+ app.connect('build-finished', on_build_finished)
+
+ return {
+ 'version': sphinx.__display_version__,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/ext/extlinks.py b/sphinx/ext/extlinks.py
index beb4a5e4b..68c1385af 100644
--- a/sphinx/ext/extlinks.py
+++ b/sphinx/ext/extlinks.py
@@ -16,21 +16,29 @@ You can also give an explicit caption, e.g. :exmpl:`Foo <foo>`.
Both, the url string and the caption string must escape ``%`` as ``%%``.
"""
+
from __future__ import annotations
+
import re
from typing import TYPE_CHECKING, Any
+
from docutils import nodes, utils
+
import sphinx
from sphinx.locale import __
from sphinx.transforms.post_transforms import SphinxPostTransform
from sphinx.util import logging, rst
from sphinx.util.nodes import split_explicit_title
+
if TYPE_CHECKING:
from collections.abc import Sequence
+
from docutils.nodes import Node, system_message
from docutils.parsers.rst.states import Inliner
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata, RoleFunction
+
logger = logging.getLogger(__name__)
@@ -40,11 +48,80 @@ class ExternalLinksChecker(SphinxPostTransform):
We treat each ``reference`` node without ``internal`` attribute as an external link.
"""
+
default_priority = 500
- def check_uri(self, refnode: nodes.reference) ->None:
+ def run(self, **kwargs: Any) -> None:
+ if not self.config.extlinks_detect_hardcoded_links:
+ return
+
+ for refnode in self.document.findall(nodes.reference):
+ self.check_uri(refnode)
+
+ def check_uri(self, refnode: nodes.reference) -> None:
"""
If the URI in ``refnode`` has a replacement in ``extlinks``,
emit a warning with a replacement suggestion.
"""
- pass
+ if 'internal' in refnode or 'refuri' not in refnode:
+ return
+
+ uri = refnode['refuri']
+ title = refnode.astext()
+
+ for alias, (base_uri, _caption) in self.app.config.extlinks.items():
+ uri_pattern = re.compile(re.escape(base_uri).replace('%s', '(?P<value>.+)'))
+
+ match = uri_pattern.match(uri)
+ if (
+ match and
+ match.groupdict().get('value') and
+ '/' not in match.groupdict()['value']
+ ):
+ # build a replacement suggestion
+ msg = __('hardcoded link %r could be replaced by an extlink '
+ '(try using %r instead)')
+ value = match.groupdict().get('value')
+ if uri != title:
+ replacement = f":{alias}:`{rst.escape(title)} <{value}>`"
+ else:
+ replacement = f":{alias}:`{value}`"
+ logger.warning(msg, uri, replacement, location=refnode)
+
+
+def make_link_role(name: str, base_url: str, caption: str) -> RoleFunction:
+ # Check whether we have base_url and caption strings have an '%s' for
+ # expansion. If not, fall back to the old behaviour and use the string as
+ # a prefix.
+ # Remark: It is an implementation detail that we use Python's %-formatting.
+ # So far we only expose ``%s`` and require quoting of ``%`` using ``%%``.
+ def role(typ: str, rawtext: str, text: str, lineno: int,
+ inliner: Inliner, options: dict[str, Any] | None = None,
+ content: Sequence[str] = (),
+ ) -> tuple[list[Node], list[system_message]]:
+ text = utils.unescape(text)
+ has_explicit_title, title, part = split_explicit_title(text)
+ full_url = base_url % part
+ if not has_explicit_title:
+ if caption is None:
+ title = full_url
+ else:
+ title = caption % part
+ pnode = nodes.reference(title, title, internal=False, refuri=full_url)
+ pnode["classes"].append(f"extlink-{name}")
+ return [pnode], []
+ return role
+
+
+def setup_link_roles(app: Sphinx) -> None:
+ for name, (base_url, caption) in app.config.extlinks.items():
+ app.add_role(name, make_link_role(name, base_url, caption))
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_config_value('extlinks', {}, 'env')
+ app.add_config_value('extlinks_detect_hardcoded_links', False, 'env')
+
+ app.connect('builder-inited', setup_link_roles)
+ app.add_post_transform(ExternalLinksChecker)
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/githubpages.py b/sphinx/ext/githubpages.py
index 67d37a5bc..aac47975a 100644
--- a/sphinx/ext/githubpages.py
+++ b/sphinx/ext/githubpages.py
@@ -1,22 +1,26 @@
"""To publish HTML docs at GitHub Pages, create .nojekyll file."""
+
from __future__ import annotations
+
import contextlib
import os
import urllib.parse
from typing import TYPE_CHECKING
+
import sphinx
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata
-def _get_domain_from_url(url: str) ->str:
+def _get_domain_from_url(url: str) -> str:
"""Get the domain from a URL."""
- pass
+ return url and urllib.parse.urlparse(url).hostname or ''
-def create_nojekyll_and_cname(app: Sphinx, env: BuildEnvironment) ->None:
+def create_nojekyll_and_cname(app: Sphinx, env: BuildEnvironment) -> None:
"""Manage the ``.nojekyll`` and ``CNAME`` files for GitHub Pages.
For HTML-format builders (e.g. 'html', 'dirhtml') we unconditionally create
@@ -31,4 +35,24 @@ def create_nojekyll_and_cname(app: Sphinx, env: BuildEnvironment) ->None:
requires a CNAME file, we remove any existing ``CNAME`` files from the
output directory.
"""
- pass
+ if app.builder.format != 'html':
+ return
+
+ app.builder.outdir.joinpath('.nojekyll').touch()
+ cname_path = os.path.join(app.builder.outdir, 'CNAME')
+
+ domain = _get_domain_from_url(app.config.html_baseurl)
+ # Filter out GitHub Pages domains, as they do not require CNAME files.
+ if domain and not domain.endswith(".github.io"):
+ with open(cname_path, 'w', encoding="utf-8") as f:
+ # NOTE: don't write a trailing newline. The `CNAME` file that's
+ # auto-generated by the GitHub UI doesn't have one.
+ f.write(domain)
+ else:
+ with contextlib.suppress(FileNotFoundError):
+ os.unlink(cname_path)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.connect('env-updated', create_nojekyll_and_cname)
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/graphviz.py b/sphinx/ext/graphviz.py
index 5333534bf..af636c747 100644
--- a/sphinx/ext/graphviz.py
+++ b/sphinx/ext/graphviz.py
@@ -1,6 +1,8 @@
"""Allow graphviz-formatted graphs to be included inline in generated documents.
"""
+
from __future__ import annotations
+
import posixpath
import re
import subprocess
@@ -11,8 +13,10 @@ from os import path
from subprocess import CalledProcessError
from typing import TYPE_CHECKING, Any, ClassVar
from urllib.parse import urlsplit, urlunsplit
+
from docutils import nodes
from docutils.parsers.rst import directives
+
import sphinx
from sphinx.errors import SphinxError
from sphinx.locale import _, __
@@ -21,8 +25,10 @@ from sphinx.util.docutils import SphinxDirective
from sphinx.util.i18n import search_image_for_language
from sphinx.util.nodes import set_source_info
from sphinx.util.osutil import ensuredir
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata, OptionSpec
@@ -31,6 +37,7 @@ if TYPE_CHECKING:
from sphinx.writers.manpage import ManualPageTranslator
from sphinx.writers.texinfo import TexinfoTranslator
from sphinx.writers.text import TextTranslator
+
logger = logging.getLogger(__name__)
@@ -40,64 +47,424 @@ class GraphvizError(SphinxError):
class ClickableMapDefinition:
"""A manipulator for clickable map file of graphviz."""
+
maptag_re = re.compile('<map id="(.*?)"')
href_re = re.compile('href=".*?"')
- def __init__(self, filename: str, content: str, dot: str='') ->None:
+ def __init__(self, filename: str, content: str, dot: str = '') -> None:
self.id: str | None = None
self.filename = filename
self.content = content.splitlines()
self.clickable: list[str] = []
+
self.parse(dot=dot)
- def generate_clickable_map(self) ->str:
+ def parse(self, dot: str) -> None:
+ matched = self.maptag_re.match(self.content[0])
+ if not matched:
+ raise GraphvizError('Invalid clickable map file found: %s' % self.filename)
+
+ self.id = matched.group(1)
+ if self.id == '%3':
+ # graphviz generates wrong ID if graph name not specified
+ # https://gitlab.com/graphviz/graphviz/issues/1327
+ hashed = sha1(dot.encode(), usedforsecurity=False).hexdigest()
+ self.id = 'grapviz%s' % hashed[-10:]
+ self.content[0] = self.content[0].replace('%3', self.id)
+
+ for line in self.content:
+ if self.href_re.search(line):
+ self.clickable.append(line)
+
+ def generate_clickable_map(self) -> str:
"""Generate clickable map tags if clickable item exists.
If not exists, this only returns empty string.
"""
- pass
+ if self.clickable:
+ return '\n'.join((self.content[0], *self.clickable, self.content[-1]))
+ else:
+ return ''
class graphviz(nodes.General, nodes.Inline, nodes.Element):
pass
+def figure_wrapper(directive: SphinxDirective, node: graphviz, caption: str) -> nodes.figure:
+ figure_node = nodes.figure('', node)
+ if 'align' in node:
+ figure_node['align'] = node.attributes.pop('align')
+
+ inodes, messages = directive.parse_inline(caption)
+ caption_node = nodes.caption(caption, '', *inodes)
+ caption_node.extend(messages)
+ set_source_info(directive, caption_node)
+ figure_node += caption_node
+ return figure_node
+
+
+def align_spec(argument: Any) -> str:
+ return directives.choice(argument, ('left', 'center', 'right'))
+
+
class Graphviz(SphinxDirective):
"""
Directive to insert arbitrary dot markup.
"""
+
has_content = True
required_arguments = 0
optional_arguments = 1
final_argument_whitespace = False
- option_spec: ClassVar[OptionSpec] = {'alt': directives.unchanged,
- 'align': align_spec, 'caption': directives.unchanged, 'layout':
- directives.unchanged, 'graphviz_dot': directives.unchanged, 'name':
- directives.unchanged, 'class': directives.class_option}
+ option_spec: ClassVar[OptionSpec] = {
+ 'alt': directives.unchanged,
+ 'align': align_spec,
+ 'caption': directives.unchanged,
+ 'layout': directives.unchanged,
+ 'graphviz_dot': directives.unchanged, # an old alias of `layout` option
+ 'name': directives.unchanged,
+ 'class': directives.class_option,
+ }
+
+ def run(self) -> list[Node]:
+ if self.arguments:
+ document = self.state.document
+ if self.content:
+ return [document.reporter.warning(
+ __('Graphviz directive cannot have both content and '
+ 'a filename argument'), line=self.lineno)]
+ argument = search_image_for_language(self.arguments[0], self.env)
+ rel_filename, filename = self.env.relfn2path(argument)
+ self.env.note_dependency(rel_filename)
+ try:
+ with open(filename, encoding='utf-8') as fp:
+ dotcode = fp.read()
+ except OSError:
+ return [document.reporter.warning(
+ __('External Graphviz file %r not found or reading '
+ 'it failed') % filename, line=self.lineno)]
+ else:
+ dotcode = '\n'.join(self.content)
+ rel_filename = None
+ if not dotcode.strip():
+ return [self.state_machine.reporter.warning(
+ __('Ignoring "graphviz" directive without content.'),
+ line=self.lineno)]
+ node = graphviz()
+ node['code'] = dotcode
+ node['options'] = {'docname': self.env.docname}
+
+ if 'graphviz_dot' in self.options:
+ node['options']['graphviz_dot'] = self.options['graphviz_dot']
+ if 'layout' in self.options:
+ node['options']['graphviz_dot'] = self.options['layout']
+ if 'alt' in self.options:
+ node['alt'] = self.options['alt']
+ if 'align' in self.options:
+ node['align'] = self.options['align']
+ if 'class' in self.options:
+ node['classes'] = self.options['class']
+ if rel_filename:
+ node['filename'] = rel_filename
+
+ if 'caption' not in self.options:
+ self.add_name(node)
+ return [node]
+ else:
+ figure = figure_wrapper(self, node, self.options['caption'])
+ self.add_name(figure)
+ return [figure]
class GraphvizSimple(SphinxDirective):
"""
Directive to insert arbitrary dot markup.
"""
+
has_content = True
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = False
- option_spec: ClassVar[OptionSpec] = {'alt': directives.unchanged,
- 'align': align_spec, 'caption': directives.unchanged, 'layout':
- directives.unchanged, 'graphviz_dot': directives.unchanged, 'name':
- directives.unchanged, 'class': directives.class_option}
+ option_spec: ClassVar[OptionSpec] = {
+ 'alt': directives.unchanged,
+ 'align': align_spec,
+ 'caption': directives.unchanged,
+ 'layout': directives.unchanged,
+ 'graphviz_dot': directives.unchanged, # an old alias of `layout` option
+ 'name': directives.unchanged,
+ 'class': directives.class_option,
+ }
+
+ def run(self) -> list[Node]:
+ node = graphviz()
+ node['code'] = '%s %s {\n%s\n}\n' % \
+ (self.name, self.arguments[0], '\n'.join(self.content))
+ node['options'] = {'docname': self.env.docname}
+ if 'graphviz_dot' in self.options:
+ node['options']['graphviz_dot'] = self.options['graphviz_dot']
+ if 'layout' in self.options:
+ node['options']['graphviz_dot'] = self.options['layout']
+ if 'alt' in self.options:
+ node['alt'] = self.options['alt']
+ if 'align' in self.options:
+ node['align'] = self.options['align']
+ if 'class' in self.options:
+ node['classes'] = self.options['class']
+ if 'caption' not in self.options:
+ self.add_name(node)
+ return [node]
+ else:
+ figure = figure_wrapper(self, node, self.options['caption'])
+ self.add_name(figure)
+ return [figure]
-def fix_svg_relative_paths(self: (HTML5Translator | LaTeXTranslator |
- TexinfoTranslator), filepath: str) ->None:
+
+def fix_svg_relative_paths(self: HTML5Translator | LaTeXTranslator | TexinfoTranslator,
+ filepath: str) -> None:
"""Change relative links in generated svg files to be relative to imgpath."""
- pass
+ tree = ET.parse(filepath) # NoQA: S314
+ root = tree.getroot()
+ ns = {'svg': 'http://www.w3.org/2000/svg', 'xlink': 'http://www.w3.org/1999/xlink'}
+ href_name = '{http://www.w3.org/1999/xlink}href'
+ modified = False
+
+ for element in chain(
+ root.findall('.//svg:image[@xlink:href]', ns),
+ root.findall('.//svg:a[@xlink:href]', ns),
+ ):
+ scheme, hostname, rel_uri, query, fragment = urlsplit(element.attrib[href_name])
+ if hostname:
+ # not a relative link
+ continue
+
+ docname = self.builder.env.path2doc(self.document["source"])
+ if docname is None:
+ # This shouldn't happen!
+ continue
+ doc_dir = self.builder.app.outdir.joinpath(docname).resolve().parent
+ old_path = doc_dir / rel_uri
+ img_path = doc_dir / self.builder.imgpath
+ new_path = path.relpath(old_path, start=img_path)
+ modified_url = urlunsplit((scheme, hostname, new_path, query, fragment))
-def render_dot(self: (HTML5Translator | LaTeXTranslator | TexinfoTranslator
- ), code: str, options: dict, format: str, prefix: str='graphviz',
- filename: (str | None)=None) ->tuple[str | None, str | None]:
+ element.set(href_name, modified_url)
+ modified = True
+
+ if modified:
+ tree.write(filepath)
+
+
+def render_dot(self: HTML5Translator | LaTeXTranslator | TexinfoTranslator,
+ code: str, options: dict, format: str,
+ prefix: str = 'graphviz', filename: str | None = None,
+ ) -> tuple[str | None, str | None]:
"""Render graphviz code into a PNG or PDF output file."""
- pass
+ graphviz_dot = options.get('graphviz_dot', self.builder.config.graphviz_dot)
+ if not graphviz_dot:
+ raise GraphvizError(
+ __('graphviz_dot executable path must be set! %r') % graphviz_dot,
+ )
+ hashkey = (code + str(options) + str(graphviz_dot) +
+ str(self.builder.config.graphviz_dot_args)).encode()
+
+ fname = f'{prefix}-{sha1(hashkey, usedforsecurity=False).hexdigest()}.{format}'
+ relfn = posixpath.join(self.builder.imgpath, fname)
+ outfn = path.join(self.builder.outdir, self.builder.imagedir, fname)
+
+ if path.isfile(outfn):
+ return relfn, outfn
+
+ if (hasattr(self.builder, '_graphviz_warned_dot') and
+ self.builder._graphviz_warned_dot.get(graphviz_dot)):
+ return None, None
+
+ ensuredir(path.dirname(outfn))
+
+ dot_args = [graphviz_dot]
+ dot_args.extend(self.builder.config.graphviz_dot_args)
+ dot_args.extend(['-T' + format, '-o' + outfn])
+
+ docname = options.get('docname', 'index')
+ if filename:
+ cwd = path.dirname(path.join(self.builder.srcdir, filename))
+ else:
+ cwd = path.dirname(path.join(self.builder.srcdir, docname))
+
+ if format == 'png':
+ dot_args.extend(['-Tcmapx', '-o%s.map' % outfn])
+
+ try:
+ ret = subprocess.run(dot_args, input=code.encode(), capture_output=True,
+ cwd=cwd, check=True)
+ except OSError:
+ logger.warning(__('dot command %r cannot be run (needed for graphviz '
+ 'output), check the graphviz_dot setting'), graphviz_dot)
+ if not hasattr(self.builder, '_graphviz_warned_dot'):
+ self.builder._graphviz_warned_dot = {} # type: ignore[union-attr]
+ self.builder._graphviz_warned_dot[graphviz_dot] = True
+ return None, None
+ except CalledProcessError as exc:
+ raise GraphvizError(__('dot exited with error:\n[stderr]\n%r\n'
+ '[stdout]\n%r') % (exc.stderr, exc.stdout)) from exc
+ if not path.isfile(outfn):
+ raise GraphvizError(__('dot did not produce an output file:\n[stderr]\n%r\n'
+ '[stdout]\n%r') % (ret.stderr, ret.stdout))
+
+ if format == 'svg':
+ fix_svg_relative_paths(self, outfn)
+
+ return relfn, outfn
+
+
+def render_dot_html(self: HTML5Translator, node: graphviz, code: str, options: dict,
+ prefix: str = 'graphviz', imgcls: str | None = None,
+ alt: str | None = None, filename: str | None = None,
+ ) -> tuple[str, str]:
+ format = self.builder.config.graphviz_output_format
+ try:
+ if format not in ('png', 'svg'):
+ raise GraphvizError(__("graphviz_output_format must be one of 'png', "
+ "'svg', but is %r") % format)
+ fname, outfn = render_dot(self, code, options, format, prefix, filename)
+ except GraphvizError as exc:
+ logger.warning(__('dot code %r: %s'), code, exc)
+ raise nodes.SkipNode from exc
+
+ classes = [imgcls, 'graphviz', *node.get('classes', [])]
+ imgcls = ' '.join(filter(None, classes))
+
+ if fname is None:
+ self.body.append(self.encode(code))
+ else:
+ if alt is None:
+ alt = node.get('alt', self.encode(code).strip())
+ if 'align' in node:
+ self.body.append('<div align="%s" class="align-%s">' %
+ (node['align'], node['align']))
+ if format == 'svg':
+ self.body.append('<div class="graphviz">')
+ self.body.append('<object data="%s" type="image/svg+xml" class="%s">\n' %
+ (fname, imgcls))
+ self.body.append('<p class="warning">%s</p>' % alt)
+ self.body.append('</object></div>\n')
+ else:
+ assert outfn is not None
+ with open(outfn + '.map', encoding='utf-8') as mapfile:
+ imgmap = ClickableMapDefinition(outfn + '.map', mapfile.read(), dot=code)
+ if imgmap.clickable:
+ # has a map
+ self.body.append('<div class="graphviz">')
+ self.body.append('<img src="%s" alt="%s" usemap="#%s" class="%s" />' %
+ (fname, alt, imgmap.id, imgcls))
+ self.body.append('</div>\n')
+ self.body.append(imgmap.generate_clickable_map())
+ else:
+ # nothing in image map
+ self.body.append('<div class="graphviz">')
+ self.body.append('<img src="%s" alt="%s" class="%s" />' %
+ (fname, alt, imgcls))
+ self.body.append('</div>\n')
+ if 'align' in node:
+ self.body.append('</div>\n')
+
+ raise nodes.SkipNode
+
+
+def html_visit_graphviz(self: HTML5Translator, node: graphviz) -> None:
+ render_dot_html(self, node, node['code'], node['options'], filename=node.get('filename'))
+
+
+def render_dot_latex(self: LaTeXTranslator, node: graphviz, code: str,
+ options: dict, prefix: str = 'graphviz', filename: str | None = None,
+ ) -> None:
+ try:
+ fname, outfn = render_dot(self, code, options, 'pdf', prefix, filename)
+ except GraphvizError as exc:
+ logger.warning(__('dot code %r: %s'), code, exc)
+ raise nodes.SkipNode from exc
+
+ is_inline = self.is_inline(node)
+
+ if not is_inline:
+ pre = ''
+ post = ''
+ if 'align' in node:
+ if node['align'] == 'left':
+ pre = '{'
+ post = r'\hspace*{\fill}}'
+ elif node['align'] == 'right':
+ pre = r'{\hspace*{\fill}'
+ post = '}'
+ elif node['align'] == 'center':
+ pre = r'{\hfill'
+ post = r'\hspace*{\fill}}'
+ self.body.append('\n%s' % pre)
+
+ self.body.append(r'\sphinxincludegraphics[]{%s}' % fname)
+
+ if not is_inline:
+ self.body.append('%s\n' % post)
+
+ raise nodes.SkipNode
+
+
+def latex_visit_graphviz(self: LaTeXTranslator, node: graphviz) -> None:
+ render_dot_latex(self, node, node['code'], node['options'], filename=node.get('filename'))
+
+
+def render_dot_texinfo(self: TexinfoTranslator, node: graphviz, code: str,
+ options: dict, prefix: str = 'graphviz') -> None:
+ try:
+ fname, outfn = render_dot(self, code, options, 'png', prefix)
+ except GraphvizError as exc:
+ logger.warning(__('dot code %r: %s'), code, exc)
+ raise nodes.SkipNode from exc
+ if fname is not None:
+ self.body.append('@image{%s,,,[graphviz],png}\n' % fname[:-4])
+ raise nodes.SkipNode
+
+
+def texinfo_visit_graphviz(self: TexinfoTranslator, node: graphviz) -> None:
+ render_dot_texinfo(self, node, node['code'], node['options'])
+
+
+def text_visit_graphviz(self: TextTranslator, node: graphviz) -> None:
+ if 'alt' in node.attributes:
+ self.add_text(_('[graph: %s]') % node['alt'])
+ else:
+ self.add_text(_('[graph]'))
+ raise nodes.SkipNode
+
+
+def man_visit_graphviz(self: ManualPageTranslator, node: graphviz) -> None:
+ if 'alt' in node.attributes:
+ self.body.append(_('[graph: %s]') % node['alt'])
+ else:
+ self.body.append(_('[graph]'))
+ raise nodes.SkipNode
+
+
+def on_config_inited(_app: Sphinx, config: Config) -> None:
+ css_path = path.join(sphinx.package_dir, 'templates', 'graphviz', 'graphviz.css')
+ config.html_static_path.append(css_path)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_node(graphviz,
+ html=(html_visit_graphviz, None),
+ latex=(latex_visit_graphviz, None),
+ texinfo=(texinfo_visit_graphviz, None),
+ text=(text_visit_graphviz, None),
+ man=(man_visit_graphviz, None))
+ app.add_directive('graphviz', Graphviz)
+ app.add_directive('graph', GraphvizSimple)
+ app.add_directive('digraph', GraphvizSimple)
+ app.add_config_value('graphviz_dot', 'dot', 'html')
+ app.add_config_value('graphviz_dot_args', [], 'html')
+ app.add_config_value('graphviz_output_format', 'png', 'html')
+ app.add_css_file('graphviz.css')
+ app.connect('config-inited', on_config_inited)
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/ifconfig.py b/sphinx/ext/ifconfig.py
index 97b3f6272..17331a0bd 100644
--- a/sphinx/ext/ifconfig.py
+++ b/sphinx/ext/ifconfig.py
@@ -13,13 +13,19 @@ The argument for ``ifconfig`` is a plain Python expression, evaluated in the
namespace of the project configuration (that is, all variables from
``conf.py`` are available.)
"""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, ClassVar
+
from docutils import nodes
+
import sphinx
from sphinx.util.docutils import SphinxDirective
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata, OptionSpec
@@ -29,8 +35,46 @@ class ifconfig(nodes.Element):
class IfConfig(SphinxDirective):
+
has_content = True
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
option_spec: ClassVar[OptionSpec] = {}
+
+ def run(self) -> list[Node]:
+ node = ifconfig()
+ node.document = self.state.document
+ self.set_source_info(node)
+ node['expr'] = self.arguments[0]
+ node += self.parse_content_to_nodes(allow_section_headings=True)
+ return [node]
+
+
+def process_ifconfig_nodes(app: Sphinx, doctree: nodes.document, docname: str) -> None:
+ ns = {confval.name: confval.value for confval in app.config}
+ ns.update(app.config.__dict__.copy())
+ ns['builder'] = app.builder.name
+ for node in list(doctree.findall(ifconfig)):
+ try:
+ res = eval(node['expr'], ns) # NoQA: S307
+ except Exception as err:
+ # handle exceptions in a clean fashion
+ from traceback import format_exception_only
+ msg = ''.join(format_exception_only(err.__class__, err))
+ newnode = doctree.reporter.error('Exception occurred in '
+ 'ifconfig expression: \n%s' %
+ msg, base_node=node)
+ node.replace_self(newnode)
+ else:
+ if not res:
+ node.replace_self([])
+ else:
+ node.replace_self(node.children)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_node(ifconfig)
+ app.add_directive('ifconfig', IfConfig)
+ app.connect('doctree-resolved', process_ifconfig_nodes)
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/imgconverter.py b/sphinx/ext/imgconverter.py
index 9c69dd116..e960dd28f 100644
--- a/sphinx/ext/imgconverter.py
+++ b/sphinx/ext/imgconverter.py
@@ -1,29 +1,96 @@
"""Image converter extension for Sphinx"""
+
from __future__ import annotations
+
import subprocess
import sys
from subprocess import CalledProcessError
from typing import TYPE_CHECKING
+
import sphinx
from sphinx.errors import ExtensionError
from sphinx.locale import __
from sphinx.transforms.post_transforms.images import ImageConverter
from sphinx.util import logging
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
class ImagemagickConverter(ImageConverter):
- conversion_rules = [('image/svg+xml', 'image/png'), ('image/gif',
- 'image/png'), ('application/pdf', 'image/png'), (
- 'application/illustrator', 'image/png'), ('image/webp', 'image/png')]
+ conversion_rules = [
+ ('image/svg+xml', 'image/png'),
+ ('image/gif', 'image/png'),
+ ('application/pdf', 'image/png'),
+ ('application/illustrator', 'image/png'),
+ ('image/webp', 'image/png'),
+ ]
- def is_available(self) ->bool:
+ def is_available(self) -> bool:
"""Confirms the converter is available or not."""
- pass
+ try:
+ args = [self.config.image_converter, '-version']
+ logger.debug('Invoking %r ...', args)
+ subprocess.run(args, capture_output=True, check=True)
+ return True
+ except OSError as exc:
+ logger.warning(__(
+ "Unable to run the image conversion command %r. "
+ "'sphinx.ext.imgconverter' requires ImageMagick by default. "
+ "Ensure it is installed, or set the 'image_converter' option "
+ "to a custom conversion command.\n\n"
+ "Traceback: %s",
+ ), self.config.image_converter, exc)
+ return False
+ except CalledProcessError as exc:
+ logger.warning(__('convert exited with error:\n'
+ '[stderr]\n%r\n[stdout]\n%r'),
+ exc.stderr, exc.stdout)
+ return False
- def convert(self, _from: str, _to: str) ->bool:
+ def convert(self, _from: str, _to: str) -> bool:
"""Converts the image to expected one."""
- pass
+ try:
+ # append an index 0 to source filename to pick up the first frame
+ # (or first page) of image (ex. Animation GIF, PDF)
+ _from += '[0]'
+
+ args = ([
+ self.config.image_converter, *self.config.image_converter_args, _from, _to,
+ ])
+ logger.debug('Invoking %r ...', args)
+ subprocess.run(args, capture_output=True, check=True)
+ return True
+ except OSError:
+ logger.warning(__('convert command %r cannot be run, '
+ 'check the image_converter setting'),
+ self.config.image_converter)
+ return False
+ except CalledProcessError as exc:
+ raise ExtensionError(__('convert exited with error:\n'
+ '[stderr]\n%r\n[stdout]\n%r') %
+ (exc.stderr, exc.stdout)) from exc
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_post_transform(ImagemagickConverter)
+ if sys.platform == 'win32':
+ # On Windows, we use Imagemagik v7 by default to avoid the trouble for
+ # convert.exe bundled with Windows.
+ app.add_config_value('image_converter', 'magick', 'env')
+ app.add_config_value('image_converter_args', ['convert'], 'env')
+ else:
+ # On other platform, we use Imagemagick v6 by default. Especially,
+ # Debian/Ubuntu are still based of v6. So we can't use "magick" command
+ # for these platforms.
+ app.add_config_value('image_converter', 'convert', 'env')
+ app.add_config_value('image_converter_args', [], 'env')
+
+ return {
+ 'version': sphinx.__display_version__,
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/ext/imgmath.py b/sphinx/ext/imgmath.py
index 666d084f5..58da35bfc 100644
--- a/sphinx/ext/imgmath.py
+++ b/sphinx/ext/imgmath.py
@@ -1,6 +1,9 @@
"""Render math in HTML via dvipng or dvisvgm."""
+
from __future__ import annotations
+
__all__ = ()
+
import base64
import contextlib
import re
@@ -11,7 +14,9 @@ from hashlib import sha1
from os import path
from subprocess import CalledProcessError
from typing import TYPE_CHECKING
+
from docutils import nodes
+
import sphinx
from sphinx import package_dir
from sphinx.errors import SphinxError
@@ -21,23 +26,29 @@ from sphinx.util.math import get_node_equation_number, wrap_displaymath
from sphinx.util.osutil import ensuredir
from sphinx.util.png import read_png_depth, write_png_depth
from sphinx.util.template import LaTeXRenderer
+
if TYPE_CHECKING:
import os
+
from docutils.nodes import Element
+
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata
from sphinx.writers.html5 import HTML5Translator
+
logger = logging.getLogger(__name__)
+
templates_path = path.join(package_dir, 'templates', 'imgmath')
class MathExtError(SphinxError):
category = 'Math extension error'
- def __init__(self, msg: str, stderr: (str | None)=None, stdout: (str |
- None)=None) ->None:
+ def __init__(
+ self, msg: str, stderr: str | None = None, stdout: str | None = None,
+ ) -> None:
if stderr:
msg += '\n[stderr]\n' + stderr
if stdout:
@@ -49,64 +60,173 @@ class InvokeError(SphinxError):
"""errors on invoking converters."""
-SUPPORT_FORMAT = 'png', 'svg'
-depth_re = re.compile('\\[\\d+ depth=(-?\\d+)\\]')
-depthsvg_re = re.compile('.*, depth=(.*)pt')
-depthsvgcomment_re = re.compile('<!-- DEPTH=(-?\\d+) -->')
+SUPPORT_FORMAT = ('png', 'svg')
+
+depth_re = re.compile(r'\[\d+ depth=(-?\d+)\]')
+depthsvg_re = re.compile(r'.*, depth=(.*)pt')
+depthsvgcomment_re = re.compile(r'<!-- DEPTH=(-?\d+) -->')
-def read_svg_depth(filename: str) ->(int | None):
+def read_svg_depth(filename: str) -> int | None:
"""Read the depth from comment at last line of SVG file
"""
- pass
+ with open(filename, encoding="utf-8") as f:
+ for line in f: # NoQA: B007
+ pass
+ # Only last line is checked
+ matched = depthsvgcomment_re.match(line)
+ if matched:
+ return int(matched.group(1))
+ return None
-def write_svg_depth(filename: str, depth: int) ->None:
+def write_svg_depth(filename: str, depth: int) -> None:
"""Write the depth to SVG file as a comment at end of file
"""
- pass
+ with open(filename, 'a', encoding="utf-8") as f:
+ f.write('\n<!-- DEPTH=%s -->' % depth)
-def generate_latex_macro(image_format: str, math: str, config: Config,
- confdir: (str | os.PathLike[str])='') ->str:
+def generate_latex_macro(image_format: str,
+ math: str,
+ config: Config,
+ confdir: str | os.PathLike[str] = '') -> str:
"""Generate LaTeX macro."""
- pass
+ variables = {
+ 'fontsize': config.imgmath_font_size,
+ 'baselineskip': int(round(config.imgmath_font_size * 1.2)),
+ 'preamble': config.imgmath_latex_preamble,
+ # the dvips option is important when imgmath_latex in ["xelatex", "tectonic"],
+ # it has no impact when imgmath_latex="latex"
+ 'tightpage': '' if image_format == 'png' else ',dvips,tightpage',
+ 'math': math,
+ }
+
+ if config.imgmath_use_preview:
+ template_name = 'preview.tex'
+ else:
+ template_name = 'template.tex'
+ for template_dir in config.templates_path:
+ for template_suffix in ('.jinja', '_t'):
+ template = path.join(confdir, template_dir, template_name + template_suffix)
+ if path.exists(template):
+ return LaTeXRenderer().render(template, variables)
-def ensure_tempdir(builder: Builder) ->str:
+ return LaTeXRenderer(templates_path).render(template_name + '.jinja', variables)
+
+
+def ensure_tempdir(builder: Builder) -> str:
"""Create temporary directory.
use only one tempdir per build -- the use of a directory is cleaner
than using temporary files, since we can clean up everything at once
just removing the whole directory (see cleanup_tempdir)
"""
- pass
+ if not hasattr(builder, '_imgmath_tempdir'):
+ builder._imgmath_tempdir = tempfile.mkdtemp() # type: ignore[attr-defined]
+
+ return builder._imgmath_tempdir # type: ignore[attr-defined]
-def compile_math(latex: str, builder: Builder) ->str:
+def compile_math(latex: str, builder: Builder) -> str:
"""Compile LaTeX macros for math to DVI."""
- pass
+ tempdir = ensure_tempdir(builder)
+ filename = path.join(tempdir, 'math.tex')
+ with open(filename, 'w', encoding='utf-8') as f:
+ f.write(latex)
+
+ imgmath_latex_name = path.basename(builder.config.imgmath_latex)
+
+ # build latex command; old versions of latex don't have the
+ # --output-directory option, so we have to manually chdir to the
+ # temp dir to run it.
+ command = [builder.config.imgmath_latex]
+ if imgmath_latex_name != 'tectonic':
+ command.append('--interaction=nonstopmode')
+ # add custom args from the config file
+ command.extend(builder.config.imgmath_latex_args)
+ command.append('math.tex')
+
+ try:
+ subprocess.run(command, capture_output=True, cwd=tempdir, check=True,
+ encoding='ascii')
+ if imgmath_latex_name in {'xelatex', 'tectonic'}:
+ return path.join(tempdir, 'math.xdv')
+ else:
+ return path.join(tempdir, 'math.dvi')
+ except OSError as exc:
+ logger.warning(__('LaTeX command %r cannot be run (needed for math '
+ 'display), check the imgmath_latex setting'),
+ builder.config.imgmath_latex)
+ raise InvokeError from exc
+ except CalledProcessError as exc:
+ msg = 'latex exited with error'
+ raise MathExtError(msg, exc.stderr, exc.stdout) from exc
-def convert_dvi_to_image(command: list[str], name: str) ->tuple[str, str]:
+def convert_dvi_to_image(command: list[str], name: str) -> tuple[str, str]:
"""Convert DVI file to specific image format."""
- pass
+ try:
+ ret = subprocess.run(command, capture_output=True, check=True, encoding='ascii')
+ return ret.stdout, ret.stderr
+ except OSError as exc:
+ logger.warning(__('%s command %r cannot be run (needed for math '
+ 'display), check the imgmath_%s setting'),
+ name, command[0], name)
+ raise InvokeError from exc
+ except CalledProcessError as exc:
+ raise MathExtError('%s exited with error' % name, exc.stderr, exc.stdout) from exc
-def convert_dvi_to_png(dvipath: str, builder: Builder, out_path: str) ->(int |
- None):
+def convert_dvi_to_png(dvipath: str, builder: Builder, out_path: str) -> int | None:
"""Convert DVI file to PNG image."""
- pass
+ name = 'dvipng'
+ command = [builder.config.imgmath_dvipng, '-o', out_path, '-T', 'tight', '-z9']
+ command.extend(builder.config.imgmath_dvipng_args)
+ if builder.config.imgmath_use_preview:
+ command.append('--depth')
+ command.append(dvipath)
+ stdout, stderr = convert_dvi_to_image(command, name)
-def convert_dvi_to_svg(dvipath: str, builder: Builder, out_path: str) ->(int |
- None):
+ depth = None
+ if builder.config.imgmath_use_preview:
+ for line in stdout.splitlines():
+ matched = depth_re.match(line)
+ if matched:
+ depth = int(matched.group(1))
+ write_png_depth(out_path, depth)
+ break
+
+ return depth
+
+
+def convert_dvi_to_svg(dvipath: str, builder: Builder, out_path: str) -> int | None:
"""Convert DVI file to SVG image."""
- pass
+ name = 'dvisvgm'
+ command = [builder.config.imgmath_dvisvgm, '-o', out_path]
+ command.extend(builder.config.imgmath_dvisvgm_args)
+ command.append(dvipath)
+
+ stdout, stderr = convert_dvi_to_image(command, name)
+
+ depth = None
+ if builder.config.imgmath_use_preview:
+ for line in stderr.splitlines(): # not stdout !
+ matched = depthsvg_re.match(line)
+ if matched:
+ depth = round(float(matched.group(1)) * 100 / 72.27) # assume 100ppi
+ write_svg_depth(out_path, depth)
+ break
+
+ return depth
-def render_math(self: HTML5Translator, math: str) ->tuple[str | None, int |
- None]:
+def render_math(
+ self: HTML5Translator,
+ math: str,
+) -> tuple[str | None, int | None]:
"""Render the LaTeX math expression *math* using latex and dvipng or
dvisvgm.
@@ -120,4 +240,170 @@ def render_math(self: HTML5Translator, math: str) ->tuple[str | None, int |
docs successfully). If the programs are there, however, they may not fail
since that indicates a problem in the math source.
"""
- pass
+ image_format = self.builder.config.imgmath_image_format.lower()
+ if image_format not in SUPPORT_FORMAT:
+ unsupported_format_msg = 'imgmath_image_format must be either "png" or "svg"'
+ raise MathExtError(unsupported_format_msg)
+
+ latex = generate_latex_macro(image_format,
+ math,
+ self.builder.config,
+ self.builder.confdir)
+
+ filename = f"{sha1(latex.encode(), usedforsecurity=False).hexdigest()}.{image_format}"
+ generated_path = path.join(self.builder.outdir, self.builder.imagedir, 'math', filename)
+ ensuredir(path.dirname(generated_path))
+ if path.isfile(generated_path):
+ if image_format == 'png':
+ depth = read_png_depth(generated_path)
+ elif image_format == 'svg':
+ depth = read_svg_depth(generated_path)
+ return generated_path, depth
+
+ # if latex or dvipng (dvisvgm) has failed once, don't bother to try again
+ if hasattr(self.builder, '_imgmath_warned_latex') or \
+ hasattr(self.builder, '_imgmath_warned_image_translator'):
+ return None, None
+
+ # .tex -> .dvi
+ try:
+ dvipath = compile_math(latex, self.builder)
+ except InvokeError:
+ self.builder._imgmath_warned_latex = True # type: ignore[attr-defined]
+ return None, None
+
+ # .dvi -> .png/.svg
+ try:
+ if image_format == 'png':
+ depth = convert_dvi_to_png(dvipath, self.builder, generated_path)
+ elif image_format == 'svg':
+ depth = convert_dvi_to_svg(dvipath, self.builder, generated_path)
+ except InvokeError:
+ self.builder._imgmath_warned_image_translator = True # type: ignore[attr-defined]
+ return None, None
+
+ return generated_path, depth
+
+
+def render_maths_to_base64(image_format: str, generated_path: str) -> str:
+ with open(generated_path, "rb") as f:
+ encoded = base64.b64encode(f.read()).decode(encoding='utf-8')
+ if image_format == 'png':
+ return f'data:image/png;base64,{encoded}'
+ if image_format == 'svg':
+ return f'data:image/svg+xml;base64,{encoded}'
+ unsupported_format_msg = 'imgmath_image_format must be either "png" or "svg"'
+ raise MathExtError(unsupported_format_msg)
+
+
+def clean_up_files(app: Sphinx, exc: Exception) -> None:
+ if exc:
+ return
+
+ if hasattr(app.builder, '_imgmath_tempdir'):
+ with contextlib.suppress(Exception):
+ shutil.rmtree(app.builder._imgmath_tempdir)
+
+ if app.builder.config.imgmath_embed:
+ # in embed mode, the images are still generated in the math output dir
+ # to be shared across workers, but are not useful to the final document
+ with contextlib.suppress(Exception):
+ shutil.rmtree(path.join(app.builder.outdir, app.builder.imagedir, 'math'))
+
+
+def get_tooltip(self: HTML5Translator, node: Element) -> str:
+ if self.builder.config.imgmath_add_tooltips:
+ return ' alt="%s"' % self.encode(node.astext()).strip()
+ return ''
+
+
+def html_visit_math(self: HTML5Translator, node: nodes.math) -> None:
+ try:
+ rendered_path, depth = render_math(self, '$' + node.astext() + '$')
+ except MathExtError as exc:
+ msg = str(exc)
+ sm = nodes.system_message(msg, type='WARNING', level=2,
+ backrefs=[], source=node.astext())
+ sm.walkabout(self)
+ logger.warning(__('display latex %r: %s'), node.astext(), msg)
+ raise nodes.SkipNode from exc
+
+ if rendered_path is None:
+ # something failed -- use text-only as a bad substitute
+ self.body.append('<span class="math">%s</span>' %
+ self.encode(node.astext()).strip())
+ else:
+ if self.builder.config.imgmath_embed:
+ image_format = self.builder.config.imgmath_image_format.lower()
+ img_src = render_maths_to_base64(image_format, rendered_path)
+ else:
+ bname = path.basename(rendered_path)
+ relative_path = path.join(self.builder.imgpath, 'math', bname)
+ img_src = relative_path.replace(path.sep, '/')
+ c = f'<img class="math" src="{img_src}"' + get_tooltip(self, node)
+ if depth is not None:
+ c += f' style="vertical-align: {-depth:d}px"'
+ self.body.append(c + '/>')
+ raise nodes.SkipNode
+
+
+def html_visit_displaymath(self: HTML5Translator, node: nodes.math_block) -> None:
+ if node['nowrap']:
+ latex = node.astext()
+ else:
+ latex = wrap_displaymath(node.astext(), None, False)
+ try:
+ rendered_path, depth = render_math(self, latex)
+ except MathExtError as exc:
+ msg = str(exc)
+ sm = nodes.system_message(msg, type='WARNING', level=2,
+ backrefs=[], source=node.astext())
+ sm.walkabout(self)
+ logger.warning(__('inline latex %r: %s'), node.astext(), msg)
+ raise nodes.SkipNode from exc
+ self.body.append(self.starttag(node, 'div', CLASS='math'))
+ self.body.append('<p>')
+ if node['number']:
+ number = get_node_equation_number(self, node)
+ self.body.append('<span class="eqno">(%s)' % number)
+ self.add_permalink_ref(node, _('Link to this equation'))
+ self.body.append('</span>')
+
+ if rendered_path is None:
+ # something failed -- use text-only as a bad substitute
+ self.body.append('<span class="math">%s</span></p>\n</div>' %
+ self.encode(node.astext()).strip())
+ else:
+ if self.builder.config.imgmath_embed:
+ image_format = self.builder.config.imgmath_image_format.lower()
+ img_src = render_maths_to_base64(image_format, rendered_path)
+ else:
+ bname = path.basename(rendered_path)
+ relative_path = path.join(self.builder.imgpath, 'math', bname)
+ img_src = relative_path.replace(path.sep, '/')
+ self.body.append(f'<img src="{img_src}"' + get_tooltip(self, node) +
+ '/></p>\n</div>')
+ raise nodes.SkipNode
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_html_math_renderer('imgmath',
+ (html_visit_math, None),
+ (html_visit_displaymath, None))
+
+ app.add_config_value('imgmath_image_format', 'png', 'html')
+ app.add_config_value('imgmath_dvipng', 'dvipng', 'html')
+ app.add_config_value('imgmath_dvisvgm', 'dvisvgm', 'html')
+ app.add_config_value('imgmath_latex', 'latex', 'html')
+ app.add_config_value('imgmath_use_preview', False, 'html')
+ app.add_config_value('imgmath_dvipng_args',
+ ['-gamma', '1.5', '-D', '110', '-bg', 'Transparent'],
+ 'html')
+ app.add_config_value('imgmath_dvisvgm_args', ['--no-fonts'], 'html')
+ app.add_config_value('imgmath_latex_args', [], 'html')
+ app.add_config_value('imgmath_latex_preamble', '', 'html')
+ app.add_config_value('imgmath_add_tooltips', True, 'html')
+ app.add_config_value('imgmath_font_size', 12, 'html')
+ app.add_config_value('imgmath_embed', False, 'html', bool)
+ app.connect('build-finished', clean_up_files)
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/inheritance_diagram.py b/sphinx/ext/inheritance_diagram.py
index 2bd75f19c..7f8d9c9c6 100644
--- a/sphinx/ext/inheritance_diagram.py
+++ b/sphinx/ext/inheritance_diagram.py
@@ -1,4 +1,4 @@
-"""Defines a docutils directive for inserting inheritance diagrams.
+r"""Defines a docutils directive for inserting inheritance diagrams.
Provide the directive with one or more classes or modules (separated
by whitespace). For modules, all of the classes in that module will
@@ -19,15 +19,17 @@ Example::
Produces a graph like the following:
A
- / \\
+ / \
B C
- / \\ /
+ / \ /
E D
The graph is inserted as a PNG+image map into HTML and a PDF in
LaTeX.
"""
+
from __future__ import annotations
+
import builtins
import hashlib
import inspect
@@ -36,41 +38,95 @@ from collections.abc import Iterable, Sequence
from importlib import import_module
from os import path
from typing import TYPE_CHECKING, Any, ClassVar, cast
+
from docutils import nodes
from docutils.parsers.rst import directives
+
import sphinx
from sphinx import addnodes
-from sphinx.ext.graphviz import figure_wrapper, graphviz, render_dot_html, render_dot_latex, render_dot_texinfo
+from sphinx.ext.graphviz import (
+ figure_wrapper,
+ graphviz,
+ render_dot_html,
+ render_dot_latex,
+ render_dot_texinfo,
+)
from sphinx.util.docutils import SphinxDirective
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata, OptionSpec
from sphinx.writers.html5 import HTML5Translator
from sphinx.writers.latex import LaTeXTranslator
from sphinx.writers.texinfo import TexinfoTranslator
-module_sig_re = re.compile(
- """^(?:([\\w.]*)\\.)? # module names
- (\\w+) \\s* $ # class/final module name
- """
- , re.VERBOSE)
-py_builtins = [obj for obj in vars(builtins).values() if inspect.isclass(obj)]
+module_sig_re = re.compile(r'''^(?:([\w.]*)\.)? # module names
+ (\w+) \s* $ # class/final module name
+ ''', re.VERBOSE)
+
+
+py_builtins = [obj for obj in vars(builtins).values()
+ if inspect.isclass(obj)]
-def try_import(objname: str) ->Any:
+
+def try_import(objname: str) -> Any:
"""Import a object or module using *name* and *currentmodule*.
*name* should be a relative name from *currentmodule* or
a fully-qualified name.
Returns imported object or module. If failed, returns None value.
"""
- pass
+ try:
+ return import_module(objname)
+ except TypeError:
+ # Relative import
+ return None
+ except ImportError:
+ matched = module_sig_re.match(objname)
+
+ if not matched:
+ return None
+
+ modname, attrname = matched.groups()
+ if modname is None:
+ return None
+ try:
+ module = import_module(modname)
+ return getattr(module, attrname, None)
+ except ImportError:
+ return None
-def import_classes(name: str, currmodule: str) ->Any:
+
+def import_classes(name: str, currmodule: str) -> Any:
"""Import a class using its fully-qualified *name*."""
- pass
+ target = None
+
+ # import class or module using currmodule
+ if currmodule:
+ target = try_import(currmodule + '.' + name)
+
+ # import class or module without currmodule
+ if target is None:
+ target = try_import(name)
+
+ if target is None:
+ raise InheritanceException(
+ 'Could not import class or module %r specified for '
+ 'inheritance diagram' % name)
+
+ if inspect.isclass(target):
+ # If imported object is a class, just return it
+ return [target]
+ elif inspect.ismodule(target):
+ # If imported object is a module, return classes defined on it
+ return [cls for cls in target.__dict__.values()
+ if inspect.isclass(cls) and cls.__module__ == target.__name__]
+ raise InheritanceException('%r specified for inheritance diagram is '
+ 'not a class or module' % name)
class InheritanceException(Exception):
@@ -84,10 +140,10 @@ class InheritanceGraph:
graphviz dot graph from them.
"""
- def __init__(self, class_names: list[str], currmodule: str,
- show_builtins: bool=False, private_bases: bool=False, parts: int=0,
- aliases: (dict[str, str] | None)=None, top_classes: Sequence[Any]=()
- ) ->None:
+ def __init__(self, class_names: list[str], currmodule: str, show_builtins: bool = False,
+ private_bases: bool = False, parts: int = 0,
+ aliases: dict[str, str] | None = None, top_classes: Sequence[Any] = (),
+ ) -> None:
"""*class_names* is a list of child classes to show bases from.
If *show_builtins* is True, then Python builtins will be shown
@@ -96,19 +152,21 @@ class InheritanceGraph:
self.class_names = class_names
classes = self._import_classes(class_names, currmodule)
self.class_info = self._class_info(classes, show_builtins,
- private_bases, parts, aliases, top_classes)
+ private_bases, parts, aliases, top_classes)
if not self.class_info:
msg = 'No classes found for inheritance diagram'
raise InheritanceException(msg)
- def _import_classes(self, class_names: list[str], currmodule: str) ->list[
- Any]:
+ def _import_classes(self, class_names: list[str], currmodule: str) -> list[Any]:
"""Import a list of classes."""
- pass
-
- def _class_info(self, classes: list[Any], show_builtins: bool,
- private_bases: bool, parts: int, aliases: (dict[str, str] | None),
- top_classes: Sequence[Any]) ->list[tuple[str, str, list[str], str]]:
+ classes: list[Any] = []
+ for name in class_names:
+ classes.extend(import_classes(name, currmodule))
+ return classes
+
+ def _class_info(self, classes: list[Any], show_builtins: bool, private_bases: bool,
+ parts: int, aliases: dict[str, str] | None, top_classes: Sequence[Any],
+ ) -> list[tuple[str, str, list[str], str]]:
"""Return name and bases for all classes that are ancestors of
*classes*.
@@ -124,32 +182,105 @@ class InheritanceGraph:
*top_classes* gives the name(s) of the top most ancestor class to
traverse to. Multiple names can be specified separated by comma.
"""
- pass
-
- def class_name(self, cls: Any, parts: int=0, aliases: (dict[str, str] |
- None)=None) ->str:
+ all_classes = {}
+
+ def recurse(cls: Any) -> None:
+ if not show_builtins and cls in py_builtins:
+ return
+ if not private_bases and cls.__name__.startswith('_'):
+ return
+
+ nodename = self.class_name(cls, parts, aliases)
+ fullname = self.class_name(cls, 0, aliases)
+
+ # Use first line of docstring as tooltip, if available
+ tooltip = None
+ try:
+ if cls.__doc__:
+ doc = cls.__doc__.strip().split("\n")[0]
+ if doc:
+ tooltip = '"%s"' % doc.replace('"', '\\"')
+ except Exception: # might raise AttributeError for strange classes
+ pass
+
+ baselist: list[str] = []
+ all_classes[cls] = (nodename, fullname, baselist, tooltip)
+
+ if fullname in top_classes:
+ return
+
+ for base in cls.__bases__:
+ if not show_builtins and base in py_builtins:
+ continue
+ if not private_bases and base.__name__.startswith('_'):
+ continue
+ baselist.append(self.class_name(base, parts, aliases))
+ if base not in all_classes:
+ recurse(base)
+
+ for cls in classes:
+ recurse(cls)
+
+ return list(all_classes.values()) # type: ignore[arg-type]
+
+ def class_name(
+ self, cls: Any, parts: int = 0, aliases: dict[str, str] | None = None,
+ ) -> str:
"""Given a class object, return a fully-qualified name.
This works for things I've tested in matplotlib so far, but may not be
completely general.
"""
- pass
-
- def get_all_class_names(self) ->list[str]:
+ module = cls.__module__
+ if module in ('__builtin__', 'builtins'):
+ fullname = cls.__name__
+ else:
+ fullname = f'{module}.{cls.__qualname__}'
+ if parts == 0:
+ result = fullname
+ else:
+ name_parts = fullname.split('.')
+ result = '.'.join(name_parts[-parts:])
+ if aliases is not None and result in aliases:
+ return aliases[result]
+ return result
+
+ def get_all_class_names(self) -> list[str]:
"""Get all of the class names involved in the graph."""
- pass
- default_graph_attrs = {'rankdir': 'LR', 'size': '"8.0, 12.0"',
- 'bgcolor': 'transparent'}
- default_node_attrs = {'shape': 'box', 'fontsize': 10, 'height': 0.25,
- 'fontname':
- '"Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans"',
- 'style': '"setlinewidth(0.5),filled"', 'fillcolor': 'white'}
- default_edge_attrs = {'arrowsize': 0.5, 'style': '"setlinewidth(0.5)"'}
-
- def generate_dot(self, name: str, urls: (dict[str, str] | None)=None,
- env: (BuildEnvironment | None)=None, graph_attrs: (dict | None)=
- None, node_attrs: (dict | None)=None, edge_attrs: (dict | None)=None
- ) ->str:
+ return [fullname for (_, fullname, _, _) in self.class_info]
+
+ # These are the default attrs for graphviz
+ default_graph_attrs = {
+ 'rankdir': 'LR',
+ 'size': '"8.0, 12.0"',
+ 'bgcolor': 'transparent',
+ }
+ default_node_attrs = {
+ 'shape': 'box',
+ 'fontsize': 10,
+ 'height': 0.25,
+ 'fontname': '"Vera Sans, DejaVu Sans, Liberation Sans, '
+ 'Arial, Helvetica, sans"',
+ 'style': '"setlinewidth(0.5),filled"',
+ 'fillcolor': 'white',
+ }
+ default_edge_attrs = {
+ 'arrowsize': 0.5,
+ 'style': '"setlinewidth(0.5)"',
+ }
+
+ def _format_node_attrs(self, attrs: dict[str, Any]) -> str:
+ return ','.join(f'{k}={v}' for k, v in sorted(attrs.items()))
+
+ def _format_graph_attrs(self, attrs: dict[str, Any]) -> str:
+ return ''.join(f'{k}={v};\n' for k, v in sorted(attrs.items()))
+
+ def generate_dot(self, name: str, urls: dict[str, str] | None = None,
+ env: BuildEnvironment | None = None,
+ graph_attrs: dict | None = None,
+ node_attrs: dict | None = None,
+ edge_attrs: dict | None = None,
+ ) -> str:
"""Generate a graphviz dot graph from the classes that were passed in
to __init__.
@@ -160,13 +291,51 @@ class InheritanceGraph:
*graph_attrs*, *node_attrs*, *edge_attrs* are dictionaries containing
key/value pairs to pass on as graphviz properties.
"""
- pass
+ if urls is None:
+ urls = {}
+ g_attrs = self.default_graph_attrs.copy()
+ n_attrs = self.default_node_attrs.copy()
+ e_attrs = self.default_edge_attrs.copy()
+ if graph_attrs is not None:
+ g_attrs.update(graph_attrs)
+ if node_attrs is not None:
+ n_attrs.update(node_attrs)
+ if edge_attrs is not None:
+ e_attrs.update(edge_attrs)
+ if env:
+ g_attrs.update(env.config.inheritance_graph_attrs)
+ n_attrs.update(env.config.inheritance_node_attrs)
+ e_attrs.update(env.config.inheritance_edge_attrs)
+
+ res: list[str] = [
+ f'digraph {name} {{\n',
+ self._format_graph_attrs(g_attrs),
+ ]
+
+ for name, fullname, bases, tooltip in sorted(self.class_info):
+ # Write the node
+ this_node_attrs = n_attrs.copy()
+ if fullname in urls:
+ this_node_attrs["URL"] = '"%s"' % urls[fullname]
+ this_node_attrs["target"] = '"_top"'
+ if tooltip:
+ this_node_attrs["tooltip"] = tooltip
+ res.append(' "%s" [%s];\n' % (name, self._format_node_attrs(this_node_attrs)))
+
+ # Write the edges
+ res.extend(
+ ' "%s" -> "%s" [%s];\n' % (base_name, name, self._format_node_attrs(e_attrs))
+ for base_name in bases
+ )
+ res.append("}\n")
+ return "".join(res)
class inheritance_diagram(graphviz):
"""
A docutils node to use as a placeholder for the inheritance diagram.
"""
+
pass
@@ -174,35 +343,152 @@ class InheritanceDiagram(SphinxDirective):
"""
Run when the inheritance_diagram directive is first encountered.
"""
+
has_content = False
required_arguments = 1
optional_arguments = 0
final_argument_whitespace = True
- option_spec: ClassVar[OptionSpec] = {'parts': int, 'private-bases':
- directives.flag, 'caption': directives.unchanged, 'top-classes':
- directives.unchanged_required}
-
-
-def html_visit_inheritance_diagram(self: HTML5Translator, node:
- inheritance_diagram) ->None:
+ option_spec: ClassVar[OptionSpec] = {
+ 'parts': int,
+ 'private-bases': directives.flag,
+ 'caption': directives.unchanged,
+ 'top-classes': directives.unchanged_required,
+ }
+
+ def run(self) -> list[Node]:
+ node = inheritance_diagram()
+ node.document = self.state.document
+ class_names = self.arguments[0].split()
+ class_role = self.env.get_domain('py').role('class')
+ # Store the original content for use as a hash
+ node['parts'] = self.options.get('parts', 0)
+ node['content'] = ', '.join(class_names)
+ node['top-classes'] = []
+ for cls in self.options.get('top-classes', '').split(','):
+ cls = cls.strip()
+ if cls:
+ node['top-classes'].append(cls)
+
+ # Create a graph starting with the list of classes
+ try:
+ graph = InheritanceGraph(
+ class_names, self.env.ref_context.get('py:module'), # type: ignore[arg-type]
+ parts=node['parts'],
+ private_bases='private-bases' in self.options,
+ aliases=self.config.inheritance_alias,
+ top_classes=node['top-classes'])
+ except InheritanceException as err:
+ return [node.document.reporter.warning(err, line=self.lineno)]
+
+ # Create xref nodes for each target of the graph's image map and
+ # add them to the doc tree so that Sphinx can resolve the
+ # references to real URLs later. These nodes will eventually be
+ # removed from the doctree after we're done with them.
+ for name in graph.get_all_class_names():
+ refnodes, x = class_role( # type: ignore[misc]
+ 'class', ':class:`%s`' % name, name, 0, self.state.inliner)
+ node.extend(refnodes)
+ # Store the graph object so we can use it to generate the
+ # dot file later
+ node['graph'] = graph
+
+ if 'caption' not in self.options:
+ self.add_name(node)
+ return [node]
+ else:
+ figure = figure_wrapper(self, node, self.options['caption'])
+ self.add_name(figure)
+ return [figure]
+
+
+def get_graph_hash(node: inheritance_diagram) -> str:
+ encoded = (node['content'] + str(node['parts'])).encode()
+ return hashlib.md5(encoded, usedforsecurity=False).hexdigest()[-10:]
+
+
+def html_visit_inheritance_diagram(self: HTML5Translator, node: inheritance_diagram) -> None:
"""
Output the graph for HTML. This will insert a PNG with clickable
image map.
"""
- pass
-
-
-def latex_visit_inheritance_diagram(self: LaTeXTranslator, node:
- inheritance_diagram) ->None:
+ graph = node['graph']
+
+ graph_hash = get_graph_hash(node)
+ name = 'inheritance%s' % graph_hash
+
+ # Create a mapping from fully-qualified class names to URLs.
+ graphviz_output_format = self.builder.env.config.graphviz_output_format.upper()
+ current_filename = path.basename(self.builder.current_docname + self.builder.out_suffix)
+ urls = {}
+ pending_xrefs = cast(Iterable[addnodes.pending_xref], node)
+ for child in pending_xrefs:
+ if child.get('refuri') is not None:
+ # Construct the name from the URI if the reference is external via intersphinx
+ if not child.get('internal', True):
+ refname = child['refuri'].rsplit('#', 1)[-1]
+ else:
+ refname = child['reftitle']
+
+ urls[refname] = child.get('refuri')
+ elif child.get('refid') is not None:
+ if graphviz_output_format == 'SVG':
+ urls[child['reftitle']] = current_filename + '#' + child.get('refid')
+ else:
+ urls[child['reftitle']] = '#' + child.get('refid')
+
+ dotcode = graph.generate_dot(name, urls, env=self.builder.env)
+ render_dot_html(self, node, dotcode, {}, 'inheritance', 'inheritance',
+ alt='Inheritance diagram of ' + node['content'])
+ raise nodes.SkipNode
+
+
+def latex_visit_inheritance_diagram(self: LaTeXTranslator, node: inheritance_diagram) -> None:
"""
Output the graph for LaTeX. This will insert a PDF.
"""
- pass
+ graph = node['graph']
+
+ graph_hash = get_graph_hash(node)
+ name = 'inheritance%s' % graph_hash
+ dotcode = graph.generate_dot(name, env=self.builder.env,
+ graph_attrs={'size': '"6.0,6.0"'})
+ render_dot_latex(self, node, dotcode, {}, 'inheritance')
+ raise nodes.SkipNode
-def texinfo_visit_inheritance_diagram(self: TexinfoTranslator, node:
- inheritance_diagram) ->None:
+
+def texinfo_visit_inheritance_diagram(self: TexinfoTranslator, node: inheritance_diagram,
+ ) -> None:
"""
Output the graph for Texinfo. This will insert a PNG.
"""
- pass
+ graph = node['graph']
+
+ graph_hash = get_graph_hash(node)
+ name = 'inheritance%s' % graph_hash
+
+ dotcode = graph.generate_dot(name, env=self.builder.env,
+ graph_attrs={'size': '"6.0,6.0"'})
+ render_dot_texinfo(self, node, dotcode, {}, 'inheritance')
+ raise nodes.SkipNode
+
+
+def skip(self: nodes.NodeVisitor, node: inheritance_diagram) -> None:
+ raise nodes.SkipNode
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.setup_extension('sphinx.ext.graphviz')
+ app.add_node(
+ inheritance_diagram,
+ latex=(latex_visit_inheritance_diagram, None),
+ html=(html_visit_inheritance_diagram, None),
+ text=(skip, None),
+ man=(skip, None),
+ texinfo=(texinfo_visit_inheritance_diagram, None))
+ app.add_directive('inheritance-diagram', InheritanceDiagram)
+ app.add_config_value('inheritance_graph_attrs', {}, '')
+ app.add_config_value('inheritance_node_attrs', {}, '')
+ app.add_config_value('inheritance_edge_attrs', {}, '')
+ app.add_config_value('inheritance_alias', {}, '')
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/intersphinx/_cli.py b/sphinx/ext/intersphinx/_cli.py
index 65410871f..25ec6ca7c 100644
--- a/sphinx/ext/intersphinx/_cli.py
+++ b/sphinx/ext/intersphinx/_cli.py
@@ -1,9 +1,45 @@
"""This module provides contains the code for intersphinx command-line utilities."""
+
from __future__ import annotations
+
import sys
+
from sphinx.ext.intersphinx._load import _fetch_inventory
-def inspect_main(argv: list[str], /) ->int:
+def inspect_main(argv: list[str], /) -> int:
"""Debug functionality to print out an inventory"""
- pass
+ if len(argv) < 1:
+ print('Print out an inventory file.\n'
+ 'Error: must specify local path or URL to an inventory file.',
+ file=sys.stderr)
+ return 1
+
+ class MockConfig:
+ intersphinx_timeout: int | None = None
+ tls_verify = False
+ tls_cacerts: str | dict[str, str] | None = None
+ user_agent: str = ''
+
+ try:
+ filename = argv[0]
+ inv_data = _fetch_inventory(
+ target_uri='',
+ inv_location=filename,
+ config=MockConfig(), # type: ignore[arg-type]
+ srcdir='' # type: ignore[arg-type]
+ )
+ for key in sorted(inv_data or {}):
+ print(key)
+ inv_entries = sorted(inv_data[key].items())
+ for entry, (_proj, _ver, url_path, display_name) in inv_entries:
+ display_name = display_name * (display_name != '-')
+ print(f' {entry:<40} {display_name:<40}: {url_path}')
+ except ValueError as exc:
+ print(exc.args[0] % exc.args[1:], file=sys.stderr)
+ return 1
+ except Exception as exc:
+ print(f'Unknown error: {exc!r}', file=sys.stderr)
+ return 1
+ else:
+ return 0
diff --git a/sphinx/ext/intersphinx/_load.py b/sphinx/ext/intersphinx/_load.py
index 021469d71..1973de379 100644
--- a/sphinx/ext/intersphinx/_load.py
+++ b/sphinx/ext/intersphinx/_load.py
@@ -1,5 +1,7 @@
"""This module contains the code for loading intersphinx inventories."""
+
from __future__ import annotations
+
import concurrent.futures
import functools
import posixpath
@@ -8,22 +10,31 @@ from operator import itemgetter
from os import path
from typing import TYPE_CHECKING
from urllib.parse import urlsplit, urlunsplit
+
from sphinx.builders.html import INVENTORY_FILENAME
from sphinx.errors import ConfigError
from sphinx.ext.intersphinx._shared import LOGGER, InventoryAdapter, _IntersphinxProject
from sphinx.locale import __
from sphinx.util import requests
from sphinx.util.inventory import InventoryFile
+
if TYPE_CHECKING:
from pathlib import Path
from typing import IO
+
from sphinx.application import Sphinx
from sphinx.config import Config
- from sphinx.ext.intersphinx._shared import IntersphinxMapping, InventoryCacheEntry, InventoryLocation, InventoryName, InventoryURI
+ from sphinx.ext.intersphinx._shared import (
+ IntersphinxMapping,
+ InventoryCacheEntry,
+ InventoryLocation,
+ InventoryName,
+ InventoryURI,
+ )
from sphinx.util.typing import Inventory
-def validate_intersphinx_mapping(app: Sphinx, config: Config) ->None:
+def validate_intersphinx_mapping(app: Sphinx, config: Config) -> None:
"""Validate and normalise :confval:`intersphinx_mapping`.
Ensure that:
@@ -35,29 +46,269 @@ def validate_intersphinx_mapping(app: Sphinx, config: Config) ->None:
* The second element of each value pair (inventory locations)
is a tuple of non-empty strings or None.
"""
- pass
+ # URIs should NOT be duplicated, otherwise different builds may use
+ # different project names (and thus, the build are no more reproducible)
+ # depending on which one is inserted last in the cache.
+ seen: dict[InventoryURI, InventoryName] = {}
+
+ errors = 0
+ for name, value in config.intersphinx_mapping.copy().items():
+ # ensure that intersphinx projects are always named
+ if not isinstance(name, str) or not name:
+ errors += 1
+ msg = __(
+ 'Invalid intersphinx project identifier `%r` in intersphinx_mapping. '
+ 'Project identifiers must be non-empty strings.'
+ )
+ LOGGER.error(msg, name)
+ del config.intersphinx_mapping[name]
+ continue
+
+ # ensure values are properly formatted
+ if not isinstance(value, (tuple | list)):
+ errors += 1
+ msg = __(
+ 'Invalid value `%r` in intersphinx_mapping[%r]. '
+ 'Expected a two-element tuple or list.'
+ )
+ LOGGER.error(msg, value, name)
+ del config.intersphinx_mapping[name]
+ continue
+ try:
+ uri, inv = value
+ except (TypeError, ValueError, Exception):
+ errors += 1
+ msg = __(
+ 'Invalid value `%r` in intersphinx_mapping[%r]. '
+ 'Values must be a (target URI, inventory locations) pair.'
+ )
+ LOGGER.error(msg, value, name)
+ del config.intersphinx_mapping[name]
+ continue
+ # ensure target URIs are non-empty and unique
+ if not uri or not isinstance(uri, str):
+ errors += 1
+ msg = __('Invalid target URI value `%r` in intersphinx_mapping[%r][0]. '
+ 'Target URIs must be unique non-empty strings.')
+ LOGGER.error(msg, uri, name)
+ del config.intersphinx_mapping[name]
+ continue
+ if uri in seen:
+ errors += 1
+ msg = __(
+ 'Invalid target URI value `%r` in intersphinx_mapping[%r][0]. '
+ 'Target URIs must be unique (other instance in intersphinx_mapping[%r]).'
+ )
+ LOGGER.error(msg, uri, name, seen[uri])
+ del config.intersphinx_mapping[name]
+ continue
+ seen[uri] = name
-def load_mappings(app: Sphinx) ->None:
+ # ensure inventory locations are None or non-empty
+ targets: list[InventoryLocation] = []
+ for target in (inv if isinstance(inv, (tuple | list)) else (inv,)):
+ if target is None or target and isinstance(target, str):
+ targets.append(target)
+ else:
+ errors += 1
+ msg = __(
+ 'Invalid inventory location value `%r` in intersphinx_mapping[%r][1]. '
+ 'Inventory locations must be non-empty strings or None.'
+ )
+ LOGGER.error(msg, target, name)
+ del config.intersphinx_mapping[name]
+ continue
+
+ config.intersphinx_mapping[name] = (name, (uri, tuple(targets)))
+
+ if errors == 1:
+ msg = __('Invalid `intersphinx_mapping` configuration (1 error).')
+ raise ConfigError(msg)
+ if errors > 1:
+ msg = __('Invalid `intersphinx_mapping` configuration (%s errors).')
+ raise ConfigError(msg % errors)
+
+
+def load_mappings(app: Sphinx) -> None:
"""Load all intersphinx mappings into the environment.
The intersphinx mappings are expected to be normalized.
"""
- pass
+ now = int(time.time())
+ inventories = InventoryAdapter(app.builder.env)
+ intersphinx_cache: dict[InventoryURI, InventoryCacheEntry] = inventories.cache
+ intersphinx_mapping: IntersphinxMapping = app.config.intersphinx_mapping
+
+ projects = []
+ for name, (uri, locations) in intersphinx_mapping.values():
+ try:
+ project = _IntersphinxProject(name=name, target_uri=uri, locations=locations)
+ except ValueError as err:
+ msg = __('An invalid intersphinx_mapping entry was added after normalisation.')
+ raise ConfigError(msg) from err
+ else:
+ projects.append(project)
+
+ expected_uris = {project.target_uri for project in projects}
+ for uri in frozenset(intersphinx_cache):
+ if intersphinx_cache[uri][0] not in intersphinx_mapping:
+ # Remove all cached entries that are no longer in `intersphinx_mapping`.
+ del intersphinx_cache[uri]
+ elif uri not in expected_uris:
+ # Remove cached entries with a different target URI
+ # than the one in `intersphinx_mapping`.
+ # This happens when the URI in `intersphinx_mapping` is changed.
+ del intersphinx_cache[uri]
+ with concurrent.futures.ThreadPoolExecutor() as pool:
+ futures = [
+ pool.submit(
+ _fetch_inventory_group,
+ project=project,
+ cache=intersphinx_cache,
+ now=now,
+ config=app.config,
+ srcdir=app.srcdir,
+ )
+ for project in projects
+ ]
+ updated = [f.result() for f in concurrent.futures.as_completed(futures)]
-def fetch_inventory(app: Sphinx, uri: InventoryURI, inv: str) ->Inventory:
+ if any(updated):
+ # clear the local inventories
+ inventories.clear()
+
+ # Duplicate values in different inventories will shadow each
+ # other; which one will override which can vary between builds.
+ #
+ # In an attempt to make this more consistent,
+ # we sort the named inventories in the cache
+ # by their name and expiry time ``(NAME, EXPIRY)``.
+ by_name_and_time = itemgetter(0, 1) # 0: name, 1: expiry
+ cache_values = sorted(intersphinx_cache.values(), key=by_name_and_time)
+ for name, _expiry, invdata in cache_values:
+ inventories.named_inventory[name] = invdata
+ for objtype, objects in invdata.items():
+ inventories.main_inventory.setdefault(objtype, {}).update(objects)
+
+
+def _fetch_inventory_group(
+ *,
+ project: _IntersphinxProject,
+ cache: dict[InventoryURI, InventoryCacheEntry],
+ now: int,
+ config: Config,
+ srcdir: Path,
+) -> bool:
+ if config.intersphinx_cache_limit < 0:
+ cache_time = now - config.intersphinx_cache_limit * 86400
+ else:
+ cache_time = 0
+
+ updated = False
+ failures = []
+
+ for location in project.locations:
+ # location is either None or a non-empty string
+ inv = f'{project.target_uri}/{INVENTORY_FILENAME}' if location is None else location
+
+ # decide whether the inventory must be read: always read local
+ # files; remote ones only if the cache time is expired
+ if (
+ '://' not in inv
+ or project.target_uri not in cache
+ or cache[project.target_uri][1] < cache_time
+ ):
+ LOGGER.info(__("loading intersphinx inventory '%s' from %s ..."),
+ project.name, _get_safe_url(inv))
+
+ try:
+ invdata = _fetch_inventory(
+ target_uri=project.target_uri,
+ inv_location=inv,
+ config=config,
+ srcdir=srcdir,
+ )
+ except Exception as err:
+ failures.append(err.args)
+ continue
+
+ if invdata:
+ cache[project.target_uri] = project.name, now, invdata
+ updated = True
+ break
+
+ if not failures:
+ pass
+ elif len(failures) < len(project.locations):
+ LOGGER.info(__('encountered some issues with some of the inventories,'
+ ' but they had working alternatives:'))
+ for fail in failures:
+ LOGGER.info(*fail)
+ else:
+ issues = '\n'.join(f[0] % f[1:] for f in failures)
+ LOGGER.warning(__('failed to reach any of the inventories '
+ 'with the following issues:') + '\n' + issues)
+ return updated
+
+
+def fetch_inventory(app: Sphinx, uri: InventoryURI, inv: str) -> Inventory:
"""Fetch, parse and return an intersphinx inventory file."""
- pass
+ return _fetch_inventory(
+ target_uri=uri,
+ inv_location=inv,
+ config=app.config,
+ srcdir=app.srcdir,
+ )
-def _fetch_inventory(*, target_uri: InventoryURI, inv_location: str, config:
- Config, srcdir: Path) ->Inventory:
+def _fetch_inventory(
+ *, target_uri: InventoryURI, inv_location: str, config: Config, srcdir: Path,
+) -> Inventory:
"""Fetch, parse and return an intersphinx inventory file."""
- pass
+ # both *target_uri* (base URI of the links to generate)
+ # and *inv_location* (actual location of the inventory file)
+ # can be local or remote URIs
+ if '://' in target_uri:
+ # case: inv URI points to remote resource; strip any existing auth
+ target_uri = _strip_basic_auth(target_uri)
+ try:
+ if '://' in inv_location:
+ f = _read_from_url(inv_location, config=config)
+ else:
+ f = open(path.join(srcdir, inv_location), 'rb') # NoQA: SIM115
+ except Exception as err:
+ err.args = ('intersphinx inventory %r not fetchable due to %s: %s',
+ inv_location, err.__class__, str(err))
+ raise
+ try:
+ if hasattr(f, 'url'):
+ new_inv_location = f.url
+ if inv_location != new_inv_location:
+ msg = __('intersphinx inventory has moved: %s -> %s')
+ LOGGER.info(msg, inv_location, new_inv_location)
+
+ if target_uri in {
+ inv_location,
+ path.dirname(inv_location),
+ path.dirname(inv_location) + '/'
+ }:
+ target_uri = path.dirname(new_inv_location)
+ with f:
+ try:
+ invdata = InventoryFile.load(f, target_uri, posixpath.join)
+ except ValueError as exc:
+ raise ValueError('unknown or unsupported inventory version: %r' % exc) from exc
+ except Exception as err:
+ err.args = ('intersphinx inventory %r not readable due to %s: %s',
+ inv_location, err.__class__.__name__, str(err))
+ raise
+ else:
+ return invdata
-def _get_safe_url(url: str) ->str:
+def _get_safe_url(url: str) -> str:
"""Gets version of *url* with basic auth passwords obscured. This function
returns results suitable for printing and logging.
@@ -69,10 +320,20 @@ def _get_safe_url(url: str) ->str:
:return: *url* with password removed
:rtype: ``str``
"""
- pass
+ parts = urlsplit(url)
+ if parts.username is None:
+ return url
+ else:
+ frags = list(parts)
+ if parts.port:
+ frags[1] = f'{parts.username}@{parts.hostname}:{parts.port}'
+ else:
+ frags[1] = f'{parts.username}@{parts.hostname}'
+
+ return urlunsplit(frags)
-def _strip_basic_auth(url: str) ->str:
+def _strip_basic_auth(url: str) -> str:
"""Returns *url* with basic auth credentials removed. Also returns the
basic auth username and password if they're present in *url*.
@@ -86,10 +347,14 @@ def _strip_basic_auth(url: str) ->str:
:return: *url* with any basic auth creds removed
:rtype: ``str``
"""
- pass
+ frags = list(urlsplit(url))
+ # swap out 'user[:pass]@hostname' for 'hostname'
+ if '@' in frags[1]:
+ frags[1] = frags[1].split('@')[1]
+ return urlunsplit(frags)
-def _read_from_url(url: str, *, config: Config) ->IO:
+def _read_from_url(url: str, *, config: Config) -> IO:
"""Reads data from *url* with an HTTP *GET*.
This function supports fetching from resources which use basic HTTP auth as
@@ -105,4 +370,12 @@ def _read_from_url(url: str, *, config: Config) ->IO:
:return: data read from resource described by *url*
:rtype: ``file``-like object
"""
- pass
+ r = requests.get(url, stream=True, timeout=config.intersphinx_timeout,
+ _user_agent=config.user_agent,
+ _tls_info=(config.tls_verify, config.tls_cacerts))
+ r.raise_for_status()
+ r.raw.url = r.url
+ # decode content-body based on the header.
+ # ref: https://github.com/psf/requests/issues/2155
+ r.raw.read = functools.partial(r.raw.read, decode_content=True)
+ return r.raw
diff --git a/sphinx/ext/intersphinx/_resolve.py b/sphinx/ext/intersphinx/_resolve.py
index 37f1281fc..35a8c12bc 100644
--- a/sphinx/ext/intersphinx/_resolve.py
+++ b/sphinx/ext/intersphinx/_resolve.py
@@ -1,10 +1,14 @@
"""This module provides logic for resolving references to intersphinx targets."""
+
from __future__ import annotations
+
import posixpath
import re
from typing import TYPE_CHECKING, cast
+
from docutils import nodes
from docutils.utils import relative_path
+
from sphinx.addnodes import pending_xref
from sphinx.deprecation import _deprecation_warning
from sphinx.errors import ExtensionError
@@ -12,12 +16,15 @@ from sphinx.ext.intersphinx._shared import LOGGER, InventoryAdapter
from sphinx.locale import _, __
from sphinx.transforms.post_transforms import ReferencesResolver
from sphinx.util.docutils import CustomReSTDispatcher, SphinxRole
+
if TYPE_CHECKING:
from collections.abc import Iterable
from types import ModuleType
from typing import Any
+
from docutils.nodes import Node, TextElement, system_message
from docutils.utils import Reporter
+
from sphinx.application import Sphinx
from sphinx.domains import Domain
from sphinx.environment import BuildEnvironment
@@ -25,30 +32,199 @@ if TYPE_CHECKING:
from sphinx.util.typing import Inventory, InventoryItem, RoleFunction
-def resolve_reference_in_inventory(env: BuildEnvironment, inv_name:
- InventoryName, node: pending_xref, contnode: TextElement) ->(nodes.
- reference | None):
+def _create_element_from_result(domain: Domain, inv_name: InventoryName | None,
+ data: InventoryItem,
+ node: pending_xref, contnode: TextElement) -> nodes.reference:
+ proj, version, uri, dispname = data
+ if '://' not in uri and node.get('refdoc'):
+ # get correct path in case of subdirectories
+ uri = posixpath.join(relative_path(node['refdoc'], '.'), uri)
+ if version:
+ reftitle = _('(in %s v%s)') % (proj, version)
+ else:
+ reftitle = _('(in %s)') % (proj,)
+ newnode = nodes.reference('', '', internal=False, refuri=uri, reftitle=reftitle)
+ if node.get('refexplicit'):
+ # use whatever title was given
+ newnode.append(contnode)
+ elif dispname == '-' or (domain.name == 'std' and node['reftype'] == 'keyword'):
+ # use whatever title was given, but strip prefix
+ title = contnode.astext()
+ if inv_name is not None and title.startswith(inv_name + ':'):
+ newnode.append(contnode.__class__(title[len(inv_name) + 1:],
+ title[len(inv_name) + 1:]))
+ else:
+ newnode.append(contnode)
+ else:
+ # else use the given display name (used for :ref:)
+ newnode.append(contnode.__class__(dispname, dispname))
+ return newnode
+
+
+def _resolve_reference_in_domain_by_target(
+ inv_name: InventoryName | None, inventory: Inventory,
+ domain: Domain, objtypes: Iterable[str],
+ target: str,
+ node: pending_xref, contnode: TextElement) -> nodes.reference | None:
+ for objtype in objtypes:
+ if objtype not in inventory:
+ # Continue if there's nothing of this kind in the inventory
+ continue
+
+ if target in inventory[objtype]:
+ # Case sensitive match, use it
+ data = inventory[objtype][target]
+ elif objtype in {'std:label', 'std:term'}:
+ # Some types require case insensitive matches:
+ # * 'term': https://github.com/sphinx-doc/sphinx/issues/9291
+ # * 'label': https://github.com/sphinx-doc/sphinx/issues/12008
+ target_lower = target.lower()
+ insensitive_matches = list(filter(lambda k: k.lower() == target_lower,
+ inventory[objtype].keys()))
+ if len(insensitive_matches) > 1:
+ data_items = {inventory[objtype][match] for match in insensitive_matches}
+ inv_descriptor = inv_name or 'main_inventory'
+ if len(data_items) == 1: # these are duplicates; relatively innocuous
+ LOGGER.debug(__("inventory '%s': duplicate matches found for %s:%s"),
+ inv_descriptor, objtype, target,
+ type='intersphinx', subtype='external', location=node)
+ else:
+ LOGGER.warning(__("inventory '%s': multiple matches found for %s:%s"),
+ inv_descriptor, objtype, target,
+ type='intersphinx', subtype='external', location=node)
+ if insensitive_matches:
+ data = inventory[objtype][insensitive_matches[0]]
+ else:
+ # No case insensitive match either, continue to the next candidate
+ continue
+ else:
+ # Could reach here if we're not a term but have a case insensitive match.
+ # This is a fix for terms specifically, but potentially should apply to
+ # other types.
+ continue
+ return _create_element_from_result(domain, inv_name, data, node, contnode)
+ return None
+
+
+def _resolve_reference_in_domain(env: BuildEnvironment,
+ inv_name: InventoryName | None, inventory: Inventory,
+ honor_disabled_refs: bool,
+ domain: Domain, objtypes: Iterable[str],
+ node: pending_xref, contnode: TextElement,
+ ) -> nodes.reference | None:
+ obj_types: dict[str, None] = {}.fromkeys(objtypes)
+
+ # we adjust the object types for backwards compatibility
+ if domain.name == 'std' and 'cmdoption' in obj_types:
+ # cmdoptions were stored as std:option until Sphinx 1.6
+ obj_types['option'] = None
+ if domain.name == 'py' and 'attribute' in obj_types:
+ # properties are stored as py:method since Sphinx 2.1
+ obj_types['method'] = None
+
+ # the inventory contains domain:type as objtype
+ domain_name = domain.name
+ obj_types = {f'{domain_name}:{obj_type}': None for obj_type in obj_types}
+
+ # now that the objtypes list is complete we can remove the disabled ones
+ if honor_disabled_refs:
+ disabled = set(env.config.intersphinx_disabled_reftypes)
+ obj_types = {obj_type: None
+ for obj_type in obj_types
+ if obj_type not in disabled}
+
+ objtypes = [*obj_types.keys()]
+
+ # without qualification
+ res = _resolve_reference_in_domain_by_target(inv_name, inventory, domain, objtypes,
+ node['reftarget'], node, contnode)
+ if res is not None:
+ return res
+
+ # try with qualification of the current scope instead
+ full_qualified_name = domain.get_full_qualified_name(node)
+ if full_qualified_name is None:
+ return None
+ return _resolve_reference_in_domain_by_target(inv_name, inventory, domain, objtypes,
+ full_qualified_name, node, contnode)
+
+
+def _resolve_reference(env: BuildEnvironment,
+ inv_name: InventoryName | None, inventory: Inventory,
+ honor_disabled_refs: bool,
+ node: pending_xref, contnode: TextElement) -> nodes.reference | None:
+ # disabling should only be done if no inventory is given
+ honor_disabled_refs = honor_disabled_refs and inv_name is None
+ intersphinx_disabled_reftypes = env.config.intersphinx_disabled_reftypes
+
+ if honor_disabled_refs and '*' in intersphinx_disabled_reftypes:
+ return None
+
+ typ = node['reftype']
+ if typ == 'any':
+ for domain_name, domain in env.domains.items():
+ if honor_disabled_refs and f'{domain_name}:*' in intersphinx_disabled_reftypes:
+ continue
+ objtypes: Iterable[str] = domain.object_types.keys()
+ res = _resolve_reference_in_domain(env, inv_name, inventory,
+ honor_disabled_refs,
+ domain, objtypes,
+ node, contnode)
+ if res is not None:
+ return res
+ return None
+ else:
+ domain_name = node.get('refdomain')
+ if not domain_name:
+ # only objects in domains are in the inventory
+ return None
+ if honor_disabled_refs and f'{domain_name}:*' in intersphinx_disabled_reftypes:
+ return None
+ domain = env.get_domain(domain_name)
+ objtypes = domain.objtypes_for_role(typ) or ()
+ if not objtypes:
+ return None
+ return _resolve_reference_in_domain(env, inv_name, inventory,
+ honor_disabled_refs,
+ domain, objtypes,
+ node, contnode)
+
+
+def inventory_exists(env: BuildEnvironment, inv_name: InventoryName) -> bool:
+ return inv_name in InventoryAdapter(env).named_inventory
+
+
+def resolve_reference_in_inventory(env: BuildEnvironment,
+ inv_name: InventoryName,
+ node: pending_xref, contnode: TextElement,
+ ) -> nodes.reference | None:
"""Attempt to resolve a missing reference via intersphinx references.
Resolution is tried in the given inventory with the target as is.
Requires ``inventory_exists(env, inv_name)``.
"""
- pass
+ assert inventory_exists(env, inv_name)
+ return _resolve_reference(env, inv_name, InventoryAdapter(env).named_inventory[inv_name],
+ False, node, contnode)
def resolve_reference_any_inventory(env: BuildEnvironment,
- honor_disabled_refs: bool, node: pending_xref, contnode: TextElement) ->(
- nodes.reference | None):
+ honor_disabled_refs: bool,
+ node: pending_xref, contnode: TextElement,
+ ) -> nodes.reference | None:
"""Attempt to resolve a missing reference via intersphinx references.
Resolution is tried with the target as is in any inventory.
"""
- pass
+ return _resolve_reference(env, None, InventoryAdapter(env).main_inventory,
+ honor_disabled_refs,
+ node, contnode)
-def resolve_reference_detect_inventory(env: BuildEnvironment, node:
- pending_xref, contnode: TextElement) ->(nodes.reference | None):
+def resolve_reference_detect_inventory(env: BuildEnvironment,
+ node: pending_xref, contnode: TextElement,
+ ) -> nodes.reference | None:
"""Attempt to resolve a missing reference via intersphinx references.
Resolution is tried first with the target as is in any inventory.
@@ -56,13 +232,28 @@ def resolve_reference_detect_inventory(env: BuildEnvironment, node:
to form ``inv_name:newtarget``. If ``inv_name`` is a named inventory, then resolution
is tried in that inventory with the new target.
"""
- pass
+ # ordinary direct lookup, use data as is
+ res = resolve_reference_any_inventory(env, True, node, contnode)
+ if res is not None:
+ return res
+
+ # try splitting the target into 'inv_name:target'
+ target = node['reftarget']
+ if ':' not in target:
+ return None
+ inv_name, newtarget = target.split(':', 1)
+ if not inventory_exists(env, inv_name):
+ return None
+ node['reftarget'] = newtarget
+ res_inv = resolve_reference_in_inventory(env, inv_name, node, contnode)
+ node['reftarget'] = target
+ return res_inv
-def missing_reference(app: Sphinx, env: BuildEnvironment, node:
- pending_xref, contnode: TextElement) ->(nodes.reference | None):
+def missing_reference(app: Sphinx, env: BuildEnvironment, node: pending_xref,
+ contnode: TextElement) -> nodes.reference | None:
"""Attempt to resolve a missing reference via intersphinx references."""
- pass
+ return resolve_reference_detect_inventory(env, node, contnode)
class IntersphinxDispatcher(CustomReSTDispatcher):
@@ -71,15 +262,120 @@ class IntersphinxDispatcher(CustomReSTDispatcher):
This enables :external:***:/:external+***: roles on parsing reST document.
"""
+ def role(
+ self, role_name: str, language_module: ModuleType, lineno: int, reporter: Reporter,
+ ) -> tuple[RoleFunction, list[system_message]]:
+ if len(role_name) > 9 and role_name.startswith(('external:', 'external+')):
+ return IntersphinxRole(role_name), []
+ else:
+ return super().role(role_name, language_module, lineno, reporter)
+
class IntersphinxRole(SphinxRole):
- _re_inv_ref = re.compile('(\\+([^:]+))?:(.*)')
+ # group 1: just for the optionality of the inventory name
+ # group 2: the inventory name (optional)
+ # group 3: the domain:role or role part
+ _re_inv_ref = re.compile(r'(\+([^:]+))?:(.*)')
- def __init__(self, orig_name: str) ->None:
+ def __init__(self, orig_name: str) -> None:
self.orig_name = orig_name
- def get_inventory_and_name_suffix(self, name: str) ->tuple[str | None, str
- ]:
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ assert self.name == self.orig_name.lower()
+ inventory, name_suffix = self.get_inventory_and_name_suffix(self.orig_name)
+ if inventory and not inventory_exists(self.env, inventory):
+ self._emit_warning(
+ __('inventory for external cross-reference not found: %r'), inventory
+ )
+ return [], []
+
+ domain_name, role_name = self._get_domain_role(name_suffix)
+
+ if role_name is None:
+ self._emit_warning(
+ __('invalid external cross-reference suffix: %r'), name_suffix
+ )
+ return [], []
+
+ # attempt to find a matching role function
+ role_func: RoleFunction | None
+
+ if domain_name is not None:
+ # the user specified a domain, so we only check that
+ if (domain := self.env.domains.get(domain_name)) is None:
+ self._emit_warning(
+ __('domain for external cross-reference not found: %r'), domain_name
+ )
+ return [], []
+ if (role_func := domain.roles.get(role_name)) is None:
+ msg = 'role for external cross-reference not found in domain %r: %r'
+ if (
+ object_types := domain.object_types.get(role_name)
+ ) is not None and object_types.roles:
+ self._emit_warning(
+ __(f'{msg} (perhaps you meant one of: %s)'),
+ domain_name,
+ role_name,
+ self._concat_strings(object_types.roles),
+ )
+ else:
+ self._emit_warning(__(msg), domain_name, role_name)
+ return [], []
+
+ else:
+ # the user did not specify a domain,
+ # so we check first the default (if available) then standard domains
+ domains: list[Domain] = []
+ if default_domain := self.env.temp_data.get('default_domain'):
+ domains.append(default_domain)
+ if (
+ std_domain := self.env.domains.get('std')
+ ) is not None and std_domain not in domains:
+ domains.append(std_domain)
+
+ role_func = None
+ for domain in domains:
+ if (role_func := domain.roles.get(role_name)) is not None:
+ domain_name = domain.name
+ break
+
+ if role_func is None or domain_name is None:
+ domains_str = self._concat_strings(d.name for d in domains)
+ msg = 'role for external cross-reference not found in domains %s: %r'
+ possible_roles: set[str] = set()
+ for d in domains:
+ if o := d.object_types.get(role_name):
+ possible_roles.update(f'{d.name}:{r}' for r in o.roles)
+ if possible_roles:
+ msg = f'{msg} (perhaps you meant one of: %s)'
+ self._emit_warning(
+ __(msg),
+ domains_str,
+ role_name,
+ self._concat_strings(possible_roles),
+ )
+ else:
+ self._emit_warning(__(msg), domains_str, role_name)
+ return [], []
+
+ result, messages = role_func(
+ f'{domain_name}:{role_name}',
+ self.rawtext,
+ self.text,
+ self.lineno,
+ self.inliner,
+ self.options,
+ self.content,
+ )
+
+ for node in result:
+ if isinstance(node, pending_xref):
+ node['intersphinx'] = True
+ node['inventory'] = inventory
+
+ return result, messages
+
+ def get_inventory_and_name_suffix(self, name: str) -> tuple[str | None, str]:
"""Extract an inventory name (if any) and ``domain+name`` suffix from a role *name*.
and the domain+name suffix.
@@ -90,21 +386,94 @@ class IntersphinxRole(SphinxRole):
- ``external:name`` -- any inventory and domain, explicit name.
- ``external:domain:name`` -- any inventory, explicit domain and name.
"""
- pass
+ assert name.startswith('external'), name
+ suffix = name[9:]
+ if name[8] == '+':
+ inv_name, suffix = suffix.split(':', 1)
+ return inv_name, suffix
+ elif name[8] == ':':
+ return None, suffix
+ else:
+ msg = f'Malformed :external: role name: {name}'
+ raise ValueError(msg)
- def _get_domain_role(self, name: str) ->tuple[str | None, str | None]:
+ def _get_domain_role(self, name: str) -> tuple[str | None, str | None]:
"""Convert the *name* string into a domain and a role name.
- If *name* contains no ``:``, return ``(None, name)``.
- If *name* contains a single ``:``, the domain/role is split on this.
- If *name* contains multiple ``:``, return ``(None, None)``.
"""
- pass
+ names = name.split(':')
+ if len(names) == 1:
+ return None, names[0]
+ elif len(names) == 2:
+ return names[0], names[1]
+ else:
+ return None, None
+
+ def _emit_warning(self, msg: str, /, *args: Any) -> None:
+ LOGGER.warning(
+ msg,
+ *args,
+ type='intersphinx',
+ subtype='external',
+ location=(self.env.docname, self.lineno),
+ )
+
+ def _concat_strings(self, strings: Iterable[str]) -> str:
+ return ', '.join(f'{s!r}' for s in sorted(strings))
+
+ # deprecated methods
- def invoke_role(self, role: tuple[str, str]) ->tuple[list[Node], list[
- system_message]]:
+ def get_role_name(self, name: str) -> tuple[str, str] | None:
+ _deprecation_warning(
+ __name__, f'{self.__class__.__name__}.get_role_name', '', remove=(9, 0)
+ )
+ names = name.split(':')
+ if len(names) == 1:
+ # role
+ default_domain = self.env.temp_data.get('default_domain')
+ domain = default_domain.name if default_domain else None
+ role = names[0]
+ elif len(names) == 2:
+ # domain:role:
+ domain = names[0]
+ role = names[1]
+ else:
+ return None
+
+ if domain and self.is_existent_role(domain, role):
+ return (domain, role)
+ elif self.is_existent_role('std', role):
+ return ('std', role)
+ else:
+ return None
+
+ def is_existent_role(self, domain_name: str, role_name: str) -> bool:
+ _deprecation_warning(
+ __name__, f'{self.__class__.__name__}.is_existent_role', '', remove=(9, 0)
+ )
+ try:
+ domain = self.env.get_domain(domain_name)
+ return role_name in domain.roles
+ except ExtensionError:
+ return False
+
+ def invoke_role(self, role: tuple[str, str]) -> tuple[list[Node], list[system_message]]:
"""Invoke the role described by a ``(domain, role name)`` pair."""
- pass
+ _deprecation_warning(
+ __name__, f'{self.__class__.__name__}.invoke_role', '', remove=(9, 0)
+ )
+ domain = self.env.get_domain(role[0])
+ if domain:
+ role_func = domain.role(role[1])
+ assert role_func is not None
+
+ return role_func(':'.join(role), self.rawtext, self.text, self.lineno,
+ self.inliner, self.options, self.content)
+ else:
+ return [], []
class IntersphinxRoleResolver(ReferencesResolver):
@@ -112,13 +481,35 @@ class IntersphinxRoleResolver(ReferencesResolver):
This resolves pending_xref nodes generated by :intersphinx:***: role.
"""
+
default_priority = ReferencesResolver.default_priority - 1
+ def run(self, **kwargs: Any) -> None:
+ for node in self.document.findall(pending_xref):
+ if 'intersphinx' not in node:
+ continue
+ contnode = cast(nodes.TextElement, node[0].deepcopy())
+ inv_name = node['inventory']
+ if inv_name is not None:
+ assert inventory_exists(self.env, inv_name)
+ newnode = resolve_reference_in_inventory(self.env, inv_name, node, contnode)
+ else:
+ newnode = resolve_reference_any_inventory(self.env, False, node, contnode)
+ if newnode is None:
+ typ = node['reftype']
+ msg = (__('external %s:%s reference target not found: %s') %
+ (node['refdomain'], typ, node['reftarget']))
+ LOGGER.warning(msg, location=node, type='ref', subtype=typ)
+ node.replace_self(contnode)
+ else:
+ node.replace_self(newnode)
+
-def install_dispatcher(app: Sphinx, docname: str, source: list[str]) ->None:
+def install_dispatcher(app: Sphinx, docname: str, source: list[str]) -> None:
"""Enable IntersphinxDispatcher.
.. note:: The installed dispatcher will be uninstalled on disabling sphinx_domain
automatically.
"""
- pass
+ dispatcher = IntersphinxDispatcher()
+ dispatcher.enable()
diff --git a/sphinx/ext/intersphinx/_shared.py b/sphinx/ext/intersphinx/_shared.py
index 56a5986a5..36c73786f 100644
--- a/sphinx/ext/intersphinx/_shared.py
+++ b/sphinx/ext/intersphinx/_shared.py
@@ -1,36 +1,68 @@
"""This module contains code shared between intersphinx modules."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any, Final, NoReturn
+
from sphinx.util import logging
+
if TYPE_CHECKING:
from collections.abc import Sequence
from typing import TypeAlias
+
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import Inventory
+
+ #: The inventory project URL to which links are resolved.
+ #:
+ #: This value is unique in :confval:`intersphinx_mapping`.
InventoryURI = str
+
+ #: The inventory (non-empty) name.
+ #:
+ #: It is unique and in bijection with an inventory remote URL.
InventoryName = str
+
+ #: A target (local or remote) containing the inventory data to fetch.
+ #:
+ #: Empty strings are not expected and ``None`` indicates the default
+ #: inventory file name :data:`~sphinx.builder.html.INVENTORY_FILENAME`.
InventoryLocation = str | None
+
+ #: Inventory cache entry. The integer field is the cache expiration time.
InventoryCacheEntry: TypeAlias = tuple[InventoryName, int, Inventory]
- IntersphinxMapping = dict[InventoryName, tuple[InventoryName, tuple[
- InventoryURI, tuple[InventoryLocation, ...]]]]
-LOGGER: Final[logging.SphinxLoggerAdapter] = logging.getLogger(
- 'sphinx.ext.intersphinx')
+
+ #: The type of :confval:`intersphinx_mapping` *after* normalisation.
+ IntersphinxMapping = dict[
+ InventoryName,
+ tuple[InventoryName, tuple[InventoryURI, tuple[InventoryLocation, ...]]],
+ ]
+
+LOGGER: Final[logging.SphinxLoggerAdapter] = logging.getLogger('sphinx.ext.intersphinx')
class _IntersphinxProject:
name: InventoryName
target_uri: InventoryURI
locations: tuple[InventoryLocation, ...]
- __slots__ = {'name':
- 'The inventory name. It is unique and in bijection with an remote inventory URL.'
- , 'target_uri':
- 'The inventory project URL to which links are resolved. It is unique and in bijection with an inventory name.'
- , 'locations':
- 'A tuple of local or remote targets containing the inventory data to fetch. None indicates the default inventory file name.'
- }
-
- def __init__(self, *, name: InventoryName, target_uri: InventoryURI,
- locations: Sequence[InventoryLocation]) ->None:
+
+ __slots__ = {
+ 'name': 'The inventory name. '
+ 'It is unique and in bijection with an remote inventory URL.',
+ 'target_uri': 'The inventory project URL to which links are resolved. '
+ 'It is unique and in bijection with an inventory name.',
+ 'locations': 'A tuple of local or remote targets containing '
+ 'the inventory data to fetch. '
+ 'None indicates the default inventory file name.',
+ }
+
+ def __init__(
+ self,
+ *,
+ name: InventoryName,
+ target_uri: InventoryURI,
+ locations: Sequence[InventoryLocation],
+ ) -> None:
if not name or not isinstance(name, str):
msg = 'name must be a non-empty string'
raise ValueError(msg)
@@ -40,33 +72,39 @@ class _IntersphinxProject:
if not locations or not isinstance(locations, tuple):
msg = 'locations must be a non-empty tuple'
raise ValueError(msg)
- if any(location is not None and (not location or not isinstance(
- location, str)) for location in locations):
+ if any(
+ location is not None and (not location or not isinstance(location, str))
+ for location in locations
+ ):
msg = 'locations must be a tuple of strings or None'
raise ValueError(msg)
object.__setattr__(self, 'name', name)
object.__setattr__(self, 'target_uri', target_uri)
object.__setattr__(self, 'locations', tuple(locations))
- def __repr__(self) ->str:
- return (
- f'{self.__class__.__name__}(name={self.name!r}, target_uri={self.target_uri!r}, locations={self.locations!r})'
- )
+ def __repr__(self) -> str:
+ return (f'{self.__class__.__name__}('
+ f'name={self.name!r}, '
+ f'target_uri={self.target_uri!r}, '
+ f'locations={self.locations!r})')
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, _IntersphinxProject):
return NotImplemented
- return (self.name == other.name and self.target_uri == other.
- target_uri and self.locations == other.locations)
+ return (
+ self.name == other.name
+ and self.target_uri == other.target_uri
+ and self.locations == other.locations
+ )
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.name, self.target_uri, self.locations))
- def __setattr__(self, key: str, value: Any) ->NoReturn:
+ def __setattr__(self, key: str, value: Any) -> NoReturn:
msg = f'{self.__class__.__name__} is immutable'
raise AttributeError(msg)
- def __delattr__(self, key: str) ->NoReturn:
+ def __delattr__(self, key: str) -> NoReturn:
msg = f'{self.__class__.__name__} is immutable'
raise AttributeError(msg)
@@ -74,15 +112,18 @@ class _IntersphinxProject:
class InventoryAdapter:
"""Inventory adapter for environment"""
- def __init__(self, env: BuildEnvironment) ->None:
+ def __init__(self, env: BuildEnvironment) -> None:
self.env = env
+
if not hasattr(env, 'intersphinx_cache'):
- self.env.intersphinx_cache = {}
- self.env.intersphinx_inventory = {}
- self.env.intersphinx_named_inventory = {}
+ # initial storage when fetching inventories before processing
+ self.env.intersphinx_cache = {} # type: ignore[attr-defined]
+
+ self.env.intersphinx_inventory = {} # type: ignore[attr-defined]
+ self.env.intersphinx_named_inventory = {} # type: ignore[attr-defined]
@property
- def cache(self) ->dict[InventoryURI, InventoryCacheEntry]:
+ def cache(self) -> dict[InventoryURI, InventoryCacheEntry]:
"""Intersphinx cache.
- Key is the URI of the remote inventory.
@@ -90,4 +131,16 @@ class InventoryAdapter:
- Element two is a time value for cache invalidation, an integer.
- Element three is the loaded remote inventory of type :class:`!Inventory`.
"""
- pass
+ return self.env.intersphinx_cache # type: ignore[attr-defined]
+
+ @property
+ def main_inventory(self) -> Inventory:
+ return self.env.intersphinx_inventory # type: ignore[attr-defined]
+
+ @property
+ def named_inventory(self) -> dict[InventoryName, Inventory]:
+ return self.env.intersphinx_named_inventory # type: ignore[attr-defined]
+
+ def clear(self) -> None:
+ self.env.intersphinx_inventory.clear() # type: ignore[attr-defined]
+ self.env.intersphinx_named_inventory.clear() # type: ignore[attr-defined]
diff --git a/sphinx/ext/linkcode.py b/sphinx/ext/linkcode.py
index 87aa9c120..93118cd67 100644
--- a/sphinx/ext/linkcode.py
+++ b/sphinx/ext/linkcode.py
@@ -1,16 +1,78 @@
"""Add external links to module code in Python object descriptions."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING
+
from docutils import nodes
+
import sphinx
from sphinx import addnodes
from sphinx.errors import SphinxError
from sphinx.locale import _
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
class LinkcodeError(SphinxError):
- category = 'linkcode error'
+ category = "linkcode error"
+
+
+def doctree_read(app: Sphinx, doctree: Node) -> None:
+ env = app.builder.env
+
+ resolve_target = getattr(env.config, 'linkcode_resolve', None)
+ if not callable(env.config.linkcode_resolve):
+ msg = 'Function `linkcode_resolve` is not given in conf.py'
+ raise LinkcodeError(msg)
+ assert resolve_target is not None # for mypy
+
+ domain_keys = {
+ 'py': ['module', 'fullname'],
+ 'c': ['names'],
+ 'cpp': ['names'],
+ 'js': ['object', 'fullname'],
+ }
+
+ for objnode in list(doctree.findall(addnodes.desc)):
+ domain = objnode.get('domain')
+ uris: set[str] = set()
+ for signode in objnode:
+ if not isinstance(signode, addnodes.desc_signature):
+ continue
+
+ # Convert signode to a specified format
+ info = {}
+ for key in domain_keys.get(domain, []):
+ value = signode.get(key)
+ if not value:
+ value = ''
+ info[key] = value
+ if not info:
+ continue
+
+ # Call user code to resolve the link
+ uri = resolve_target(domain, info)
+ if not uri:
+ # no source
+ continue
+
+ if uri in uris or not uri:
+ # only one link per name, please
+ continue
+ uris.add(uri)
+
+ inline = nodes.inline('', _('[source]'), classes=['viewcode-link'])
+ onlynode = addnodes.only(expr='html')
+ onlynode += nodes.reference('', '', inline, internal=False, refuri=uri)
+ signode += onlynode
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.connect('doctree-read', doctree_read)
+ app.add_config_value('linkcode_resolve', None, '')
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/mathjax.py b/sphinx/ext/mathjax.py
index 807fbc111..f67f87fed 100644
--- a/sphinx/ext/mathjax.py
+++ b/sphinx/ext/mathjax.py
@@ -4,19 +4,124 @@ This requires the MathJax JavaScript library on your webserver/computer.
.. _MathJax: https://www.mathjax.org/
"""
+
from __future__ import annotations
+
import json
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes
+
import sphinx
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.domains.math import MathDomain
from sphinx.errors import ExtensionError
from sphinx.locale import _
from sphinx.util.math import get_node_equation_number
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
from sphinx.writers.html5 import HTML5Translator
+
+# more information for mathjax secure url is here:
+# https://docs.mathjax.org/en/latest/web/start.html#using-mathjax-from-a-content-delivery-network-cdn
MATHJAX_URL = 'https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js'
+
logger = sphinx.util.logging.getLogger(__name__)
+
+
+def html_visit_math(self: HTML5Translator, node: nodes.math) -> None:
+ self.body.append(self.starttag(node, 'span', '', CLASS='math notranslate nohighlight'))
+ self.body.append(self.builder.config.mathjax_inline[0] +
+ self.encode(node.astext()) +
+ self.builder.config.mathjax_inline[1] + '</span>')
+ raise nodes.SkipNode
+
+
+def html_visit_displaymath(self: HTML5Translator, node: nodes.math_block) -> None:
+ self.body.append(self.starttag(node, 'div', CLASS='math notranslate nohighlight'))
+ if node['nowrap']:
+ self.body.append(self.encode(node.astext()))
+ self.body.append('</div>')
+ raise nodes.SkipNode
+
+ # necessary to e.g. set the id property correctly
+ if node['number']:
+ number = get_node_equation_number(self, node)
+ self.body.append('<span class="eqno">(%s)' % number)
+ self.add_permalink_ref(node, _('Link to this equation'))
+ self.body.append('</span>')
+ self.body.append(self.builder.config.mathjax_display[0])
+ parts = [prt for prt in node.astext().split('\n\n') if prt.strip()]
+ if len(parts) > 1: # Add alignment if there are more than 1 equation
+ self.body.append(r' \begin{align}\begin{aligned}')
+ for i, part in enumerate(parts):
+ part = self.encode(part)
+ if r'\\' in part:
+ self.body.append(r'\begin{split}' + part + r'\end{split}')
+ else:
+ self.body.append(part)
+ if i < len(parts) - 1: # append new line if not the last equation
+ self.body.append(r'\\')
+ if len(parts) > 1: # Add alignment if there are more than 1 equation
+ self.body.append(r'\end{aligned}\end{align} ')
+ self.body.append(self.builder.config.mathjax_display[1])
+ self.body.append('</div>\n')
+ raise nodes.SkipNode
+
+
+def install_mathjax(app: Sphinx, pagename: str, templatename: str, context: dict[str, Any],
+ event_arg: Any) -> None:
+ if (
+ app.builder.format != 'html' or
+ app.builder.math_renderer_name != 'mathjax' # type: ignore[attr-defined]
+ ):
+ return
+ if not app.config.mathjax_path:
+ msg = 'mathjax_path config value must be set for the mathjax extension to work'
+ raise ExtensionError(msg)
+
+ domain = cast(MathDomain, app.env.get_domain('math'))
+ builder = cast(StandaloneHTMLBuilder, app.builder)
+ if app.registry.html_assets_policy == 'always' or domain.has_equations(pagename):
+ # Enable mathjax only if equations exists
+ if app.config.mathjax2_config:
+ if app.config.mathjax_path == MATHJAX_URL:
+ logger.warning(
+ 'mathjax_config/mathjax2_config does not work '
+ 'for the current MathJax version, use mathjax3_config instead')
+ body = 'MathJax.Hub.Config(%s)' % json.dumps(app.config.mathjax2_config)
+ builder.add_js_file('', type='text/x-mathjax-config', body=body)
+ if app.config.mathjax3_config:
+ body = 'window.MathJax = %s' % json.dumps(app.config.mathjax3_config)
+ builder.add_js_file('', body=body)
+
+ options = {}
+ if app.config.mathjax_options:
+ options.update(app.config.mathjax_options)
+ if 'async' not in options and 'defer' not in options:
+ if app.config.mathjax3_config:
+ # Load MathJax v3 via "defer" method
+ options['defer'] = 'defer'
+ else:
+ # Load other MathJax via "async" method
+ options['async'] = 'async'
+ builder.add_js_file(app.config.mathjax_path, **options)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_html_math_renderer('mathjax',
+ (html_visit_math, None),
+ (html_visit_displaymath, None))
+
+ app.add_config_value('mathjax_path', MATHJAX_URL, 'html')
+ app.add_config_value('mathjax_options', {}, 'html')
+ app.add_config_value('mathjax_inline', [r'\(', r'\)'], 'html')
+ app.add_config_value('mathjax_display', [r'\[', r'\]'], 'html')
+ app.add_config_value('mathjax_config', None, 'html')
+ app.add_config_value('mathjax2_config', lambda c: c.mathjax_config, 'html')
+ app.add_config_value('mathjax3_config', None, 'html')
+ app.connect('html-page-context', install_mathjax)
+
+ return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py
index 9ee30cfc5..cc3c7f4ce 100644
--- a/sphinx/ext/napoleon/docstring.py
+++ b/sphinx/ext/napoleon/docstring.py
@@ -1,5 +1,7 @@
"""Classes for docstring parsing and formatting."""
+
from __future__ import annotations
+
import collections
import contextlib
import inspect
@@ -7,33 +9,47 @@ import re
from functools import partial
from itertools import starmap
from typing import TYPE_CHECKING, Any
+
from sphinx.locale import _, __
from sphinx.util import logging
from sphinx.util.typing import get_type_hints, stringify_annotation
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
+
from sphinx.application import Sphinx
from sphinx.config import Config as SphinxConfig
+
logger = logging.getLogger(__name__)
-_directive_regex = re.compile('\\.\\. \\S+::')
-_google_section_regex = re.compile('^(\\s|\\w)+:\\s*$')
-_google_typed_arg_regex = re.compile('(.+?)\\(\\s*(.*[^\\s]+)\\s*\\)')
-_numpy_section_regex = re.compile('^[=\\-`:\\\'"~^_*+#<>]{2,}\\s*$')
-_single_colon_regex = re.compile('(?<!:):(?!:)')
+
+_directive_regex = re.compile(r'\.\. \S+::')
+_google_section_regex = re.compile(r'^(\s|\w)+:\s*$')
+_google_typed_arg_regex = re.compile(r'(.+?)\(\s*(.*[^\s]+)\s*\)')
+_numpy_section_regex = re.compile(r'^[=\-`:\'"~^_*+#<>]{2,}\s*$')
+_single_colon_regex = re.compile(r'(?<!:):(?!:)')
_xref_or_code_regex = re.compile(
- '((?::(?:[a-zA-Z0-9]+[\\-_+:.])*[a-zA-Z0-9]+:`.+?`)|(?:``.+?``)|(?::meta .+:.*)|(?:`.+?\\s*(?<!\\x00)<.*?>`))'
- )
+ r'((?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)|'
+ r'(?:``.+?``)|'
+ r'(?::meta .+:.*)|'
+ r'(?:`.+?\s*(?<!\x00)<.*?>`))')
_xref_regex = re.compile(
- '(?:(?::(?:[a-zA-Z0-9]+[\\-_+:.])*[a-zA-Z0-9]+:)?`.+?`)')
-_bullet_list_regex = re.compile('^(\\*|\\+|\\-)(\\s+\\S|\\s*$)')
+ r'(?:(?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:)?`.+?`)',
+)
+_bullet_list_regex = re.compile(r'^(\*|\+|\-)(\s+\S|\s*$)')
_enumerated_list_regex = re.compile(
- '^(?P<paren>\\()?(\\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])(?(paren)\\)|\\.)(\\s+\\S|\\s*$)'
- )
+ r'^(?P<paren>\()?'
+ r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])'
+ r'(?(paren)\)|\.)(\s+\S|\s*$)')
_token_regex = re.compile(
- '(,\\sor\\s|\\sor\\s|\\sof\\s|:\\s|\\sto\\s|,\\sand\\s|\\sand\\s|,\\s|[{]|[}]|"(?:\\\\"|[^"])*"|\'(?:\\\\\'|[^\'])*\')'
- )
-_default_regex = re.compile('^default[^_0-9A-Za-z].*$')
-_SINGLETONS = 'None', 'True', 'False', 'Ellipsis'
+ r"(,\sor\s|\sor\s|\sof\s|:\s|\sto\s|,\sand\s|\sand\s|,\s"
+ r"|[{]|[}]"
+ r'|"(?:\\"|[^"])*"'
+ r"|'(?:\\'|[^'])*')",
+)
+_default_regex = re.compile(
+ r"^default[^_0-9A-Za-z].*$",
+)
+_SINGLETONS = ("None", "True", "False", "Ellipsis")
class Deque(collections.deque):
@@ -42,20 +58,30 @@ class Deque(collections.deque):
The `.Deque.get` and `.Deque.next` methods are added.
"""
+
sentinel = object()
- def get(self, n: int) ->Any:
+ def get(self, n: int) -> Any:
"""
Return the nth element of the stack, or ``self.sentinel`` if n is
greater than the stack size.
"""
- pass
+ return self[n] if n < len(self) else self.sentinel
+
+ def next(self) -> Any:
+ if self:
+ return super().popleft()
+ else:
+ raise StopIteration
-def _convert_type_spec(_type: str, translations: (dict[str, str] | None)=None
- ) ->str:
+def _convert_type_spec(_type: str, translations: dict[str, str] | None = None) -> str:
"""Convert type specification to reference in reST."""
- pass
+ if translations is not None and _type in translations:
+ return translations[_type]
+ if _type == 'None':
+ return ':py:obj:`None`'
+ return f':py:class:`{_type}`'
class GoogleDocstring:
@@ -120,13 +146,20 @@ class GoogleDocstring:
<BLANKLINE>
"""
- _name_rgx = re.compile(
- '^\\s*((?::(?P<role>\\S+):)?`(?P<name>~?[a-zA-Z0-9_.-]+)`| (?P<name2>~?[a-zA-Z0-9_.-]+))\\s*'
- , re.VERBOSE)
- def __init__(self, docstring: (str | list[str]), config: (SphinxConfig |
- None)=None, app: (Sphinx | None)=None, what: str='', name: str='',
- obj: Any=None, options: Any=None) ->None:
+ _name_rgx = re.compile(r"^\s*((?::(?P<role>\S+):)?`(?P<name>~?[a-zA-Z0-9_.-]+)`|"
+ r" (?P<name2>~?[a-zA-Z0-9_.-]+))\s*", re.VERBOSE)
+
+ def __init__(
+ self,
+ docstring: str | list[str],
+ config: SphinxConfig | None = None,
+ app: Sphinx | None = None,
+ what: str = '',
+ name: str = '',
+ obj: Any = None,
+ options: Any = None,
+ ) -> None:
self._app = app
if config:
self._config = config
@@ -134,7 +167,9 @@ class GoogleDocstring:
self._config = app.config
else:
from sphinx.ext.napoleon import Config
- self._config = Config()
+
+ self._config = Config() # type: ignore[assignment]
+
if not what:
if inspect.isclass(obj):
what = 'class'
@@ -144,6 +179,7 @@ class GoogleDocstring:
what = 'function'
else:
what = 'object'
+
self._what = what
self._name = name
self._obj = obj
@@ -159,43 +195,48 @@ class GoogleDocstring:
if not hasattr(self, '_directive_sections'):
self._directive_sections: list[str] = []
if not hasattr(self, '_sections'):
- self._sections: dict[str, Callable] = {'args': self.
- _parse_parameters_section, 'arguments': self.
- _parse_parameters_section, 'attention': partial(self.
- _parse_admonition, 'attention'), 'attributes': self.
- _parse_attributes_section, 'caution': partial(self.
- _parse_admonition, 'caution'), 'danger': partial(self.
- _parse_admonition, 'danger'), 'error': partial(self.
- _parse_admonition, 'error'), 'example': self.
- _parse_examples_section, 'examples': self.
- _parse_examples_section, 'hint': partial(self.
- _parse_admonition, 'hint'), 'important': partial(self.
- _parse_admonition, 'important'), 'keyword args': self.
- _parse_keyword_arguments_section, 'keyword arguments': self
- ._parse_keyword_arguments_section, 'methods': self.
- _parse_methods_section, 'note': partial(self.
- _parse_admonition, 'note'), 'notes': self.
- _parse_notes_section, 'other parameters': self.
- _parse_other_parameters_section, 'parameters': self.
- _parse_parameters_section, 'receive': self.
- _parse_receives_section, 'receives': self.
- _parse_receives_section, 'return': self.
- _parse_returns_section, 'returns': self.
- _parse_returns_section, 'raise': self._parse_raises_section,
- 'raises': self._parse_raises_section, 'references': self.
- _parse_references_section, 'see also': self.
- _parse_see_also_section, 'tip': partial(self.
- _parse_admonition, 'tip'), 'todo': partial(self.
- _parse_admonition, 'todo'), 'warning': partial(self.
- _parse_admonition, 'warning'), 'warnings': partial(self.
- _parse_admonition, 'warning'), 'warn': self.
- _parse_warns_section, 'warns': self._parse_warns_section,
- 'yield': self._parse_yields_section, 'yields': self.
- _parse_yields_section}
+ self._sections: dict[str, Callable] = {
+ 'args': self._parse_parameters_section,
+ 'arguments': self._parse_parameters_section,
+ 'attention': partial(self._parse_admonition, 'attention'),
+ 'attributes': self._parse_attributes_section,
+ 'caution': partial(self._parse_admonition, 'caution'),
+ 'danger': partial(self._parse_admonition, 'danger'),
+ 'error': partial(self._parse_admonition, 'error'),
+ 'example': self._parse_examples_section,
+ 'examples': self._parse_examples_section,
+ 'hint': partial(self._parse_admonition, 'hint'),
+ 'important': partial(self._parse_admonition, 'important'),
+ 'keyword args': self._parse_keyword_arguments_section,
+ 'keyword arguments': self._parse_keyword_arguments_section,
+ 'methods': self._parse_methods_section,
+ 'note': partial(self._parse_admonition, 'note'),
+ 'notes': self._parse_notes_section,
+ 'other parameters': self._parse_other_parameters_section,
+ 'parameters': self._parse_parameters_section,
+ 'receive': self._parse_receives_section,
+ 'receives': self._parse_receives_section,
+ 'return': self._parse_returns_section,
+ 'returns': self._parse_returns_section,
+ 'raise': self._parse_raises_section,
+ 'raises': self._parse_raises_section,
+ 'references': self._parse_references_section,
+ 'see also': self._parse_see_also_section,
+ 'tip': partial(self._parse_admonition, 'tip'),
+ 'todo': partial(self._parse_admonition, 'todo'),
+ 'warning': partial(self._parse_admonition, 'warning'),
+ 'warnings': partial(self._parse_admonition, 'warning'),
+ 'warn': self._parse_warns_section,
+ 'warns': self._parse_warns_section,
+ 'yield': self._parse_yields_section,
+ 'yields': self._parse_yields_section,
+ }
+
self._load_custom_sections()
+
self._parse()
- def __str__(self) ->str:
+ def __str__(self) -> str:
"""Return the parsed docstring in reStructuredText format.
Returns
@@ -206,7 +247,7 @@ class GoogleDocstring:
"""
return '\n'.join(self.lines())
- def lines(self) ->list[str]:
+ def lines(self) -> list[str]:
"""Return the parsed lines of the docstring in reStructuredText format.
Returns
@@ -215,7 +256,812 @@ class GoogleDocstring:
The lines of the docstring in a list.
"""
- pass
+ return self._parsed_lines
+
+ def _consume_indented_block(self, indent: int = 1) -> list[str]:
+ lines = []
+ line = self._lines.get(0)
+ while (
+ not self._is_section_break() and
+ (not line or self._is_indented(line, indent))
+ ):
+ lines.append(self._lines.next())
+ line = self._lines.get(0)
+ return lines
+
+ def _consume_contiguous(self) -> list[str]:
+ lines = []
+ while (self._lines and
+ self._lines.get(0) and
+ not self._is_section_header()):
+ lines.append(self._lines.next())
+ return lines
+
+ def _consume_empty(self) -> list[str]:
+ lines = []
+ line = self._lines.get(0)
+ while self._lines and not line:
+ lines.append(self._lines.next())
+ line = self._lines.get(0)
+ return lines
+
+ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False,
+ ) -> tuple[str, str, list[str]]:
+ line = self._lines.next()
+
+ before, colon, after = self._partition_field_on_colon(line)
+ _name, _type, _desc = before, '', after
+
+ if parse_type:
+ match = _google_typed_arg_regex.match(before)
+ if match:
+ _name = match.group(1).strip()
+ _type = match.group(2)
+
+ _name = self._escape_args_and_kwargs(_name)
+
+ if prefer_type and not _type:
+ _type, _name = _name, _type
+
+ if _type and self._config.napoleon_preprocess_types:
+ _type = _convert_type_spec(_type, self._config.napoleon_type_aliases or {})
+
+ indent = self._get_indent(line) + 1
+ _descs = [_desc, *self._dedent(self._consume_indented_block(indent))]
+ _descs = self.__class__(_descs, self._config).lines()
+ return _name, _type, _descs
+
+ def _consume_fields(self, parse_type: bool = True, prefer_type: bool = False,
+ multiple: bool = False) -> list[tuple[str, str, list[str]]]:
+ self._consume_empty()
+ fields: list[tuple[str, str, list[str]]] = []
+ while not self._is_section_break():
+ _name, _type, _desc = self._consume_field(parse_type, prefer_type)
+ if multiple and _name:
+ fields.extend((name.strip(), _type, _desc) for name in _name.split(","))
+ elif _name or _type or _desc:
+ fields.append((_name, _type, _desc))
+ return fields
+
+ def _consume_inline_attribute(self) -> tuple[str, list[str]]:
+ line = self._lines.next()
+ _type, colon, _desc = self._partition_field_on_colon(line)
+ if not colon or not _desc:
+ _type, _desc = _desc, _type
+ _desc += colon
+ _descs = [_desc, *self._dedent(self._consume_to_end())]
+ _descs = self.__class__(_descs, self._config).lines()
+ return _type, _descs
+
+ def _consume_returns_section(self, preprocess_types: bool = False,
+ ) -> list[tuple[str, str, list[str]]]:
+ lines = self._dedent(self._consume_to_next_section())
+ if lines:
+ before, colon, after = self._partition_field_on_colon(lines[0])
+ _name, _type, _desc = '', '', lines
+
+ if colon:
+ if after:
+ _desc = [after] + lines[1:]
+ else:
+ _desc = lines[1:]
+
+ _type = before
+
+ if (_type and preprocess_types and
+ self._config.napoleon_preprocess_types):
+ _type = _convert_type_spec(_type, self._config.napoleon_type_aliases or {})
+
+ _desc = self.__class__(_desc, self._config).lines()
+ return [(_name, _type, _desc)]
+ else:
+ return []
+
+ def _consume_usage_section(self) -> list[str]:
+ lines = self._dedent(self._consume_to_next_section())
+ return lines
+
+ def _consume_section_header(self) -> str:
+ section = self._lines.next()
+ stripped_section = section.strip(':')
+ if stripped_section.lower() in self._sections:
+ section = stripped_section
+ return section
+
+ def _consume_to_end(self) -> list[str]:
+ lines = []
+ while self._lines:
+ lines.append(self._lines.next())
+ return lines
+
+ def _consume_to_next_section(self) -> list[str]:
+ self._consume_empty()
+ lines = []
+ while not self._is_section_break():
+ lines.append(self._lines.next())
+ return lines + self._consume_empty()
+
+ def _dedent(self, lines: list[str], full: bool = False) -> list[str]:
+ if full:
+ return [line.lstrip() for line in lines]
+ else:
+ min_indent = self._get_min_indent(lines)
+ return [line[min_indent:] for line in lines]
+
+ def _escape_args_and_kwargs(self, name: str) -> str:
+ if name.endswith('_') and getattr(self._config, 'strip_signature_backslash', False):
+ name = name[:-1] + r'\_'
+
+ if name[:2] == '**':
+ return r'\*\*' + name[2:]
+ elif name[:1] == '*':
+ return r'\*' + name[1:]
+ else:
+ return name
+
+ def _fix_field_desc(self, desc: list[str]) -> list[str]:
+ if self._is_list(desc):
+ desc = ['', *desc]
+ elif desc[0].endswith('::'):
+ desc_block = desc[1:]
+ indent = self._get_indent(desc[0])
+ block_indent = self._get_initial_indent(desc_block)
+ if block_indent > indent:
+ desc = ['', *desc]
+ else:
+ desc = ['', desc[0], *self._indent(desc_block, 4)]
+ return desc
+
+ def _format_admonition(self, admonition: str, lines: list[str]) -> list[str]:
+ lines = self._strip_empty(lines)
+ if len(lines) == 1:
+ return [f'.. {admonition}:: {lines[0].strip()}', '']
+ elif lines:
+ lines = self._indent(self._dedent(lines), 3)
+ return ['.. %s::' % admonition, '', *lines, '']
+ else:
+ return ['.. %s::' % admonition, '']
+
+ def _format_block(
+ self, prefix: str, lines: list[str], padding: str | None = None,
+ ) -> list[str]:
+ if lines:
+ if padding is None:
+ padding = ' ' * len(prefix)
+ result_lines = []
+ for i, line in enumerate(lines):
+ if i == 0:
+ result_lines.append((prefix + line).rstrip())
+ elif line:
+ result_lines.append(padding + line)
+ else:
+ result_lines.append('')
+ return result_lines
+ else:
+ return [prefix]
+
+ def _format_docutils_params(self, fields: list[tuple[str, str, list[str]]],
+ field_role: str = 'param', type_role: str = 'type',
+ ) -> list[str]:
+ lines = []
+ for _name, _type, _desc in fields:
+ _desc = self._strip_empty(_desc)
+ if any(_desc):
+ _desc = self._fix_field_desc(_desc)
+ field = f':{field_role} {_name}: '
+ lines.extend(self._format_block(field, _desc))
+ else:
+ lines.append(f':{field_role} {_name}:')
+
+ if _type:
+ lines.append(f':{type_role} {_name}: {_type}')
+ return [*lines, '']
+
+ def _format_field(self, _name: str, _type: str, _desc: list[str]) -> list[str]:
+ _desc = self._strip_empty(_desc)
+ has_desc = any(_desc)
+ separator = ' -- ' if has_desc else ''
+ if _name:
+ if _type:
+ if '`' in _type:
+ field = f'**{_name}** ({_type}){separator}'
+ else:
+ field = f'**{_name}** (*{_type}*){separator}'
+ else:
+ field = f'**{_name}**{separator}'
+ elif _type:
+ if '`' in _type:
+ field = f'{_type}{separator}'
+ else:
+ field = f'*{_type}*{separator}'
+ else:
+ field = ''
+
+ if has_desc:
+ _desc = self._fix_field_desc(_desc)
+ if _desc[0]:
+ return [field + _desc[0]] + _desc[1:]
+ else:
+ return [field, *_desc]
+ else:
+ return [field]
+
+ def _format_fields(self, field_type: str, fields: list[tuple[str, str, list[str]]],
+ ) -> list[str]:
+ field_type = ':%s:' % field_type.strip()
+ padding = ' ' * len(field_type)
+ multi = len(fields) > 1
+ lines: list[str] = []
+ for _name, _type, _desc in fields:
+ field = self._format_field(_name, _type, _desc)
+ if multi:
+ if lines:
+ lines.extend(self._format_block(padding + ' * ', field))
+ else:
+ lines.extend(self._format_block(field_type + ' * ', field))
+ else:
+ lines.extend(self._format_block(field_type + ' ', field))
+ if lines and lines[-1]:
+ lines.append('')
+ return lines
+
+ def _get_current_indent(self, peek_ahead: int = 0) -> int:
+ line = self._lines.get(peek_ahead)
+ while line is not self._lines.sentinel:
+ if line:
+ return self._get_indent(line)
+ peek_ahead += 1
+ line = self._lines.get(peek_ahead)
+ return 0
+
+ def _get_indent(self, line: str) -> int:
+ for i, s in enumerate(line):
+ if not s.isspace():
+ return i
+ return len(line)
+
+ def _get_initial_indent(self, lines: list[str]) -> int:
+ for line in lines:
+ if line:
+ return self._get_indent(line)
+ return 0
+
+ def _get_min_indent(self, lines: list[str]) -> int:
+ min_indent = None
+ for line in lines:
+ if line:
+ indent = self._get_indent(line)
+ if min_indent is None or indent < min_indent:
+ min_indent = indent
+ return min_indent or 0
+
+ def _indent(self, lines: list[str], n: int = 4) -> list[str]:
+ return [(' ' * n) + line for line in lines]
+
+ def _is_indented(self, line: str, indent: int = 1) -> bool:
+ for i, s in enumerate(line): # NoQA: SIM110
+ if i >= indent:
+ return True
+ elif not s.isspace():
+ return False
+ return False
+
+ def _is_list(self, lines: list[str]) -> bool:
+ if not lines:
+ return False
+ if _bullet_list_regex.match(lines[0]):
+ return True
+ if _enumerated_list_regex.match(lines[0]):
+ return True
+ if len(lines) < 2 or lines[0].endswith('::'):
+ return False
+ indent = self._get_indent(lines[0])
+ next_indent = indent
+ for line in lines[1:]:
+ if line:
+ next_indent = self._get_indent(line)
+ break
+ return next_indent > indent
+
+ def _is_section_header(self) -> bool:
+ section = self._lines.get(0).lower()
+ match = _google_section_regex.match(section)
+ if match and section.strip(':') in self._sections:
+ header_indent = self._get_indent(section)
+ section_indent = self._get_current_indent(peek_ahead=1)
+ return section_indent > header_indent
+ elif self._directive_sections:
+ if _directive_regex.match(section):
+ for directive_section in self._directive_sections:
+ if section.startswith(directive_section):
+ return True
+ return False
+
+ def _is_section_break(self) -> bool:
+ line = self._lines.get(0)
+ return (not self._lines or
+ self._is_section_header() or
+ (self._is_in_section and
+ line and
+ not self._is_indented(line, self._section_indent)))
+
+ def _load_custom_sections(self) -> None:
+ if self._config.napoleon_custom_sections is not None:
+ for entry in self._config.napoleon_custom_sections:
+ if isinstance(entry, str):
+ # if entry is just a label, add to sections list,
+ # using generic section logic.
+ self._sections[entry.lower()] = self._parse_custom_generic_section
+ else:
+ # otherwise, assume entry is container;
+ if entry[1] == "params_style":
+ self._sections[entry[0].lower()] = \
+ self._parse_custom_params_style_section
+ elif entry[1] == "returns_style":
+ self._sections[entry[0].lower()] = \
+ self._parse_custom_returns_style_section
+ else:
+ # [0] is new section, [1] is the section to alias.
+ # in the case of key mismatch, just handle as generic section.
+ self._sections[entry[0].lower()] = \
+ self._sections.get(entry[1].lower(),
+ self._parse_custom_generic_section)
+
+ def _parse(self) -> None:
+ self._parsed_lines = self._consume_empty()
+
+ if self._name and self._what in ('attribute', 'data', 'property'):
+ res: list[str] = []
+ with contextlib.suppress(StopIteration):
+ res = self._parse_attribute_docstring()
+
+ self._parsed_lines.extend(res)
+ return
+
+ while self._lines:
+ if self._is_section_header():
+ try:
+ section = self._consume_section_header()
+ self._is_in_section = True
+ self._section_indent = self._get_current_indent()
+ if _directive_regex.match(section):
+ lines = [section, *self._consume_to_next_section()]
+ else:
+ lines = self._sections[section.lower()](section)
+ finally:
+ self._is_in_section = False
+ self._section_indent = 0
+ else:
+ if not self._parsed_lines:
+ lines = self._consume_contiguous() + self._consume_empty()
+ else:
+ lines = self._consume_to_next_section()
+ self._parsed_lines.extend(lines)
+
+ def _parse_admonition(self, admonition: str, section: str) -> list[str]:
+ # type (str, str) -> List[str]
+ lines = self._consume_to_next_section()
+ return self._format_admonition(admonition, lines)
+
+ def _parse_attribute_docstring(self) -> list[str]:
+ _type, _desc = self._consume_inline_attribute()
+ lines = self._format_field('', '', _desc)
+ if _type:
+ lines.extend(['', ':type: %s' % _type])
+ return lines
+
+ def _parse_attributes_section(self, section: str) -> list[str]:
+ lines = []
+ for _name, _type, _desc in self._consume_fields():
+ if not _type:
+ _type = self._lookup_annotation(_name)
+ if self._config.napoleon_use_ivar:
+ field = ':ivar %s: ' % _name
+ lines.extend(self._format_block(field, _desc))
+ if _type:
+ lines.append(f':vartype {_name}: {_type}')
+ else:
+ lines.append('.. attribute:: ' + _name)
+ if self._opt:
+ if 'no-index' in self._opt or 'noindex' in self._opt:
+ lines.append(' :no-index:')
+ lines.append('')
+
+ fields = self._format_field('', '', _desc)
+ lines.extend(self._indent(fields, 3))
+ if _type:
+ lines.append('')
+ lines.extend(self._indent([':type: %s' % _type], 3))
+ lines.append('')
+ if self._config.napoleon_use_ivar:
+ lines.append('')
+ return lines
+
+ def _parse_examples_section(self, section: str) -> list[str]:
+ labels = {
+ 'example': _('Example'),
+ 'examples': _('Examples'),
+ }
+ use_admonition = self._config.napoleon_use_admonition_for_examples
+ label = labels.get(section.lower(), section)
+ return self._parse_generic_section(label, use_admonition)
+
+ def _parse_custom_generic_section(self, section: str) -> list[str]:
+ # for now, no admonition for simple custom sections
+ return self._parse_generic_section(section, False)
+
+ def _parse_custom_params_style_section(self, section: str) -> list[str]:
+ return self._format_fields(section, self._consume_fields())
+
+ def _parse_custom_returns_style_section(self, section: str) -> list[str]:
+ fields = self._consume_returns_section(preprocess_types=True)
+ return self._format_fields(section, fields)
+
+ def _parse_usage_section(self, section: str) -> list[str]:
+ header = ['.. rubric:: Usage:', '']
+ block = ['.. code-block:: python', '']
+ lines = self._consume_usage_section()
+ lines = self._indent(lines, 3)
+ return header + block + lines + ['']
+
+ def _parse_generic_section(self, section: str, use_admonition: bool) -> list[str]:
+ lines = self._strip_empty(self._consume_to_next_section())
+ lines = self._dedent(lines)
+ if use_admonition:
+ header = '.. admonition:: %s' % section
+ lines = self._indent(lines, 3)
+ else:
+ header = '.. rubric:: %s' % section
+ if lines:
+ return [header, '', *lines, '']
+ else:
+ return [header, '']
+
+ def _parse_keyword_arguments_section(self, section: str) -> list[str]:
+ fields = self._consume_fields()
+ if self._config.napoleon_use_keyword:
+ return self._format_docutils_params(
+ fields,
+ field_role="keyword",
+ type_role="kwtype")
+ else:
+ return self._format_fields(_('Keyword Arguments'), fields)
+
+ def _parse_methods_section(self, section: str) -> list[str]:
+ lines: list[str] = []
+ for _name, _type, _desc in self._consume_fields(parse_type=False):
+ lines.append('.. method:: %s' % _name)
+ if self._opt:
+ if 'no-index' in self._opt or 'noindex' in self._opt:
+ lines.append(' :no-index:')
+ if _desc:
+ lines.extend(['', *self._indent(_desc, 3)])
+ lines.append('')
+ return lines
+
+ def _parse_notes_section(self, section: str) -> list[str]:
+ use_admonition = self._config.napoleon_use_admonition_for_notes
+ return self._parse_generic_section(_('Notes'), use_admonition)
+
+ def _parse_other_parameters_section(self, section: str) -> list[str]:
+ if self._config.napoleon_use_param:
+ # Allow to declare multiple parameters at once (ex: x, y: int)
+ fields = self._consume_fields(multiple=True)
+ return self._format_docutils_params(fields)
+ else:
+ fields = self._consume_fields()
+ return self._format_fields(_('Other Parameters'), fields)
+
+ def _parse_parameters_section(self, section: str) -> list[str]:
+ if self._config.napoleon_use_param:
+ # Allow to declare multiple parameters at once (ex: x, y: int)
+ fields = self._consume_fields(multiple=True)
+ return self._format_docutils_params(fields)
+ else:
+ fields = self._consume_fields()
+ return self._format_fields(_('Parameters'), fields)
+
+ def _parse_raises_section(self, section: str) -> list[str]:
+ fields = self._consume_fields(parse_type=False, prefer_type=True)
+ lines: list[str] = []
+ for _name, _type, _desc in fields:
+ m = self._name_rgx.match(_type)
+ if m and m.group('name'):
+ _type = m.group('name')
+ elif _xref_regex.match(_type):
+ pos = _type.find('`')
+ _type = _type[pos + 1:-1]
+ _type = ' ' + _type if _type else ''
+ _desc = self._strip_empty(_desc)
+ _descs = ' ' + '\n '.join(_desc) if any(_desc) else ''
+ lines.append(f':raises{_type}:{_descs}')
+ if lines:
+ lines.append('')
+ return lines
+
+ def _parse_receives_section(self, section: str) -> list[str]:
+ if self._config.napoleon_use_param:
+ # Allow to declare multiple parameters at once (ex: x, y: int)
+ fields = self._consume_fields(multiple=True)
+ return self._format_docutils_params(fields)
+ else:
+ fields = self._consume_fields()
+ return self._format_fields(_('Receives'), fields)
+
+ def _parse_references_section(self, section: str) -> list[str]:
+ use_admonition = self._config.napoleon_use_admonition_for_references
+ return self._parse_generic_section(_('References'), use_admonition)
+
+ def _parse_returns_section(self, section: str) -> list[str]:
+ fields = self._consume_returns_section()
+ multi = len(fields) > 1
+ use_rtype = False if multi else self._config.napoleon_use_rtype
+ lines: list[str] = []
+
+ for _name, _type, _desc in fields:
+ if use_rtype:
+ field = self._format_field(_name, '', _desc)
+ else:
+ field = self._format_field(_name, _type, _desc)
+
+ if multi:
+ if lines:
+ lines.extend(self._format_block(' * ', field))
+ else:
+ lines.extend(self._format_block(':returns: * ', field))
+ else:
+ if any(field): # only add :returns: if there's something to say
+ lines.extend(self._format_block(':returns: ', field))
+ if _type and use_rtype:
+ lines.extend([':rtype: %s' % _type, ''])
+ if lines and lines[-1]:
+ lines.append('')
+ return lines
+
+ def _parse_see_also_section(self, section: str) -> list[str]:
+ return self._parse_admonition('seealso', section)
+
+ def _parse_warns_section(self, section: str) -> list[str]:
+ return self._format_fields(_('Warns'), self._consume_fields())
+
+ def _parse_yields_section(self, section: str) -> list[str]:
+ fields = self._consume_returns_section(preprocess_types=True)
+ return self._format_fields(_('Yields'), fields)
+
+ def _partition_field_on_colon(self, line: str) -> tuple[str, str, str]:
+ before_colon = []
+ after_colon = []
+ colon = ''
+ found_colon = False
+ for i, source in enumerate(_xref_or_code_regex.split(line)):
+ if found_colon:
+ after_colon.append(source)
+ else:
+ m = _single_colon_regex.search(source)
+ if (i % 2) == 0 and m:
+ found_colon = True
+ colon = source[m.start(): m.end()]
+ before_colon.append(source[:m.start()])
+ after_colon.append(source[m.end():])
+ else:
+ before_colon.append(source)
+
+ return ("".join(before_colon).strip(),
+ colon,
+ "".join(after_colon).strip())
+
+ def _strip_empty(self, lines: list[str]) -> list[str]:
+ if lines:
+ start = -1
+ for i, line in enumerate(lines):
+ if line:
+ start = i
+ break
+ if start == -1:
+ lines = []
+ end = -1
+ for i in reversed(range(len(lines))):
+ line = lines[i]
+ if line:
+ end = i
+ break
+ if start > 0 or end + 1 < len(lines):
+ lines = lines[start:end + 1]
+ return lines
+
+ def _lookup_annotation(self, _name: str) -> str:
+ if self._config.napoleon_attr_annotations:
+ if self._what in ("module", "class", "exception") and self._obj:
+ # cache the class annotations
+ if not hasattr(self, "_annotations"):
+ localns = getattr(self._config, "autodoc_type_aliases", {})
+ localns.update(getattr(
+ self._config, "napoleon_type_aliases", {},
+ ) or {})
+ self._annotations = get_type_hints(self._obj, None, localns)
+ if _name in self._annotations:
+ return stringify_annotation(self._annotations[_name],
+ 'fully-qualified-except-typing')
+ # No annotation found
+ return ""
+
+
+def _recombine_set_tokens(tokens: list[str]) -> list[str]:
+ token_queue = collections.deque(tokens)
+ keywords = ("optional", "default")
+
+ def takewhile_set(tokens: collections.deque[str]) -> Iterator[str]:
+ open_braces = 0
+ previous_token = None
+ while True:
+ try:
+ token = tokens.popleft()
+ except IndexError:
+ break
+
+ if token == ", ":
+ previous_token = token
+ continue
+
+ if not token.strip():
+ continue
+
+ if token in keywords:
+ tokens.appendleft(token)
+ if previous_token is not None:
+ tokens.appendleft(previous_token)
+ break
+
+ if previous_token is not None:
+ yield previous_token
+ previous_token = None
+
+ if token == "{":
+ open_braces += 1
+ elif token == "}":
+ open_braces -= 1
+
+ yield token
+
+ if open_braces == 0:
+ break
+
+ def combine_set(tokens: collections.deque[str]) -> Iterator[str]:
+ while True:
+ try:
+ token = tokens.popleft()
+ except IndexError:
+ break
+
+ if token == "{":
+ tokens.appendleft("{")
+ yield "".join(takewhile_set(tokens))
+ else:
+ yield token
+
+ return list(combine_set(token_queue))
+
+
+def _tokenize_type_spec(spec: str) -> list[str]:
+ def postprocess(item: str) -> list[str]:
+ if _default_regex.match(item):
+ default = item[:7]
+ # can't be separated by anything other than a single space
+ # for now
+ other = item[8:]
+
+ return [default, " ", other]
+ else:
+ return [item]
+
+ tokens = [
+ item
+ for raw_token in _token_regex.split(spec)
+ for item in postprocess(raw_token)
+ if item
+ ]
+ return tokens
+
+
+def _token_type(token: str, location: str | None = None) -> str:
+ def is_numeric(token: str) -> bool:
+ try:
+ # use complex to make sure every numeric value is detected as literal
+ complex(token)
+ except ValueError:
+ return False
+ else:
+ return True
+
+ if token.startswith(" ") or token.endswith(" "):
+ type_ = "delimiter"
+ elif (
+ is_numeric(token) or
+ (token.startswith("{") and token.endswith("}")) or
+ (token.startswith('"') and token.endswith('"')) or
+ (token.startswith("'") and token.endswith("'"))
+ ):
+ type_ = "literal"
+ elif token.startswith("{"):
+ logger.warning(
+ __("invalid value set (missing closing brace): %s"),
+ token,
+ location=location,
+ )
+ type_ = "literal"
+ elif token.endswith("}"):
+ logger.warning(
+ __("invalid value set (missing opening brace): %s"),
+ token,
+ location=location,
+ )
+ type_ = "literal"
+ elif token.startswith(("'", '"')):
+ logger.warning(
+ __("malformed string literal (missing closing quote): %s"),
+ token,
+ location=location,
+ )
+ type_ = "literal"
+ elif token.endswith(("'", '"')):
+ logger.warning(
+ __("malformed string literal (missing opening quote): %s"),
+ token,
+ location=location,
+ )
+ type_ = "literal"
+ elif token in ("optional", "default"):
+ # default is not a official keyword (yet) but supported by the
+ # reference implementation (numpydoc) and widely used
+ type_ = "control"
+ elif _xref_regex.match(token):
+ type_ = "reference"
+ else:
+ type_ = "obj"
+
+ return type_
+
+
+def _convert_numpy_type_spec(
+ _type: str, location: str | None = None, translations: dict | None = None,
+) -> str:
+ if translations is None:
+ translations = {}
+
+ def convert_obj(obj: str, translations: dict[str, str], default_translation: str) -> str:
+ translation = translations.get(obj, obj)
+
+ # use :class: (the default) only if obj is not a standard singleton
+ if translation in _SINGLETONS and default_translation == ":class:`%s`":
+ default_translation = ":obj:`%s`"
+ elif translation == "..." and default_translation == ":class:`%s`":
+ # allow referencing the builtin ...
+ default_translation = ":obj:`%s <Ellipsis>`"
+
+ if _xref_regex.match(translation) is None:
+ translation = default_translation % translation
+
+ return translation
+
+ tokens = _tokenize_type_spec(_type)
+ combined_tokens = _recombine_set_tokens(tokens)
+ types = [
+ (token, _token_type(token, location))
+ for token in combined_tokens
+ ]
+
+ converters = {
+ "literal": lambda x: "``%s``" % x,
+ "obj": lambda x: convert_obj(x, translations, ":class:`%s`"),
+ "control": lambda x: "*%s*" % x,
+ "delimiter": lambda x: x,
+ "reference": lambda x: x,
+ }
+
+ converted = "".join(converters.get(type_)(token) # type: ignore[misc]
+ for token, type_ in types)
+
+ return converted
class NumpyDocstring(GoogleDocstring):
@@ -312,13 +1158,109 @@ class NumpyDocstring(GoogleDocstring):
"""
- def __init__(self, docstring: (str | list[str]), config: (SphinxConfig |
- None)=None, app: (Sphinx | None)=None, what: str='', name: str='',
- obj: Any=None, options: Any=None) ->None:
+ def __init__(
+ self,
+ docstring: str | list[str],
+ config: SphinxConfig | None = None,
+ app: Sphinx | None = None,
+ what: str = '',
+ name: str = '',
+ obj: Any = None,
+ options: Any = None,
+ ) -> None:
self._directive_sections = ['.. index::']
super().__init__(docstring, config, app, what, name, obj, options)
- def _parse_numpydoc_see_also_section(self, content: list[str]) ->list[str]:
+ def _get_location(self) -> str | None:
+ try:
+ filepath = inspect.getfile(self._obj) if self._obj is not None else None
+ except TypeError:
+ filepath = None
+ name = self._name
+
+ if filepath is None and name is None:
+ return None
+ elif filepath is None:
+ filepath = ""
+
+ return f"{filepath}:docstring of {name}"
+
+ def _escape_args_and_kwargs(self, name: str) -> str:
+ func = super()._escape_args_and_kwargs
+
+ if ", " in name:
+ return ", ".join(map(func, name.split(", ")))
+ else:
+ return func(name)
+
+ def _consume_field(self, parse_type: bool = True, prefer_type: bool = False,
+ ) -> tuple[str, str, list[str]]:
+ line = self._lines.next()
+ if parse_type:
+ _name, _, _type = self._partition_field_on_colon(line)
+ else:
+ _name, _type = line, ''
+ _name, _type = _name.strip(), _type.strip()
+ _name = self._escape_args_and_kwargs(_name)
+
+ if parse_type and not _type:
+ _type = self._lookup_annotation(_name)
+
+ if prefer_type and not _type:
+ _type, _name = _name, _type
+
+ if self._config.napoleon_preprocess_types:
+ _type = _convert_numpy_type_spec(
+ _type,
+ location=self._get_location(),
+ translations=self._config.napoleon_type_aliases or {},
+ )
+
+ indent = self._get_indent(line) + 1
+ _desc = self._dedent(self._consume_indented_block(indent))
+ _desc = self.__class__(_desc, self._config).lines()
+ return _name, _type, _desc
+
+ def _consume_returns_section(self, preprocess_types: bool = False,
+ ) -> list[tuple[str, str, list[str]]]:
+ return self._consume_fields(prefer_type=True)
+
+ def _consume_section_header(self) -> str:
+ section = self._lines.next()
+ if not _directive_regex.match(section):
+ # Consume the header underline
+ self._lines.next()
+ return section
+
+ def _is_section_break(self) -> bool:
+ line1, line2 = self._lines.get(0), self._lines.get(1)
+ return (not self._lines or
+ self._is_section_header() or
+ (line1 == line2 == '') or
+ (self._is_in_section and
+ line1 and
+ not self._is_indented(line1, self._section_indent)))
+
+ def _is_section_header(self) -> bool:
+ section, underline = self._lines.get(0), self._lines.get(1)
+ section = section.lower()
+ if section in self._sections and isinstance(underline, str):
+ return bool(_numpy_section_regex.match(underline))
+ elif self._directive_sections:
+ if _directive_regex.match(section):
+ for directive_section in self._directive_sections:
+ if section.startswith(directive_section):
+ return True
+ return False
+
+ def _parse_see_also_section(self, section: str) -> list[str]:
+ lines = self._consume_to_next_section()
+ try:
+ return self._parse_numpydoc_see_also_section(lines)
+ except ValueError:
+ return self._format_admonition('seealso', lines)
+
+ def _parse_numpydoc_see_also_section(self, content: list[str]) -> list[str]:
"""
Derived from the NumpyDoc implementation of _parse_see_also.
@@ -330,4 +1272,94 @@ class NumpyDocstring(GoogleDocstring):
func_name1, func_name2, :meth:`func_name`, func_name3
"""
- pass
+ items: list[tuple[str, list[str], str | None]] = []
+
+ def parse_item_name(text: str) -> tuple[str, str | None]:
+ """Match ':role:`name`' or 'name'"""
+ m = self._name_rgx.match(text)
+ if m:
+ g = m.groups()
+ if g[1] is None:
+ return g[3], None
+ else:
+ return g[2], g[1]
+ raise ValueError("%s is not a item name" % text)
+
+ def push_item(name: str | None, rest: list[str]) -> None:
+ if not name:
+ return
+ name, role = parse_item_name(name)
+ items.append((name, rest.copy(), role))
+ rest.clear()
+
+ def translate(
+ func: str, description: list[str], role: str | None,
+ ) -> tuple[str, list[str], str | None]:
+ translations = self._config.napoleon_type_aliases
+ if role is not None or not translations:
+ return func, description, role
+
+ translated = translations.get(func, func)
+ match = self._name_rgx.match(translated)
+ if not match:
+ return translated, description, role
+
+ groups = match.groupdict()
+ role = groups["role"]
+ new_func = groups["name"] or groups["name2"]
+
+ return new_func, description, role
+
+ current_func = None
+ rest: list[str] = []
+
+ for line in content:
+ if not line.strip():
+ continue
+
+ m = self._name_rgx.match(line)
+ if m and line[m.end():].strip().startswith(':'):
+ push_item(current_func, rest)
+ current_func, line = line[:m.end()], line[m.end():]
+ rest = [line.split(':', 1)[1].strip()]
+ if not rest[0]:
+ rest = []
+ elif not line.startswith(' '):
+ push_item(current_func, rest)
+ current_func = None
+ if ',' in line:
+ for func in line.split(','):
+ if func.strip():
+ push_item(func, [])
+ elif line.strip():
+ current_func = line
+ elif current_func is not None:
+ rest.append(line.strip())
+ push_item(current_func, rest)
+
+ if not items:
+ return []
+
+ # apply type aliases
+ items = list(starmap(translate, items))
+
+ lines: list[str] = []
+ last_had_desc = True
+ for name, desc, role in items:
+ if role:
+ link = f':{role}:`{name}`'
+ else:
+ link = ':obj:`%s`' % name
+ if desc or last_had_desc:
+ lines += ['']
+ lines += [link]
+ else:
+ lines[-1] += ", %s" % link
+ if desc:
+ lines += self._indent([' '.join(desc)])
+ last_had_desc = True
+ else:
+ last_had_desc = False
+ lines += ['']
+
+ return self._format_admonition('seealso', lines)
diff --git a/sphinx/ext/todo.py b/sphinx/ext/todo.py
index 423cd2039..ac8e17fd5 100644
--- a/sphinx/ext/todo.py
+++ b/sphinx/ext/todo.py
@@ -4,13 +4,17 @@ Inclusion of todos can be switched of by a configuration variable.
The todolist directive collects all todos of your project and lists them along
with a backlink to the original location.
"""
+
from __future__ import annotations
+
import functools
import operator
from typing import TYPE_CHECKING, Any, ClassVar, cast
+
from docutils import nodes
from docutils.parsers.rst import directives
from docutils.parsers.rst.directives.admonitions import BaseAdmonition
+
import sphinx
from sphinx import addnodes
from sphinx.domains import Domain
@@ -18,13 +22,16 @@ from sphinx.errors import NoUri
from sphinx.locale import _, __
from sphinx.util import logging, texescape
from sphinx.util.docutils import SphinxDirective, new_document
+
if TYPE_CHECKING:
from docutils.nodes import Element, Node
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata, OptionSpec
from sphinx.writers.html5 import HTML5Translator
from sphinx.writers.latex import LaTeXTranslator
+
logger = logging.getLogger(__name__)
@@ -36,46 +43,211 @@ class todolist(nodes.General, nodes.Element):
pass
-class Todo(BaseAdmonition, SphinxDirective):
+class Todo(BaseAdmonition, SphinxDirective): # type: ignore[misc]
"""
A todo entry, displayed (if configured) in the form of an admonition.
"""
+
node_class = todo_node
has_content = True
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = False
- option_spec: ClassVar[OptionSpec] = {'class': directives.class_option,
- 'name': directives.unchanged}
+ option_spec: ClassVar[OptionSpec] = {
+ 'class': directives.class_option,
+ 'name': directives.unchanged,
+ }
+
+ def run(self) -> list[Node]:
+ if not self.options.get('class'):
+ self.options['class'] = ['admonition-todo']
+
+ (todo,) = super().run()
+ if isinstance(todo, nodes.system_message):
+ return [todo]
+ elif isinstance(todo, todo_node):
+ todo.insert(0, nodes.title(text=_('Todo')))
+ todo['docname'] = self.env.docname
+ self.add_name(todo)
+ self.set_source_info(todo)
+ self.state.document.note_explicit_target(todo)
+ return [todo]
+ else:
+ raise RuntimeError # never reached here
class TodoDomain(Domain):
name = 'todo'
label = 'todo'
+ @property
+ def todos(self) -> dict[str, list[todo_node]]:
+ return self.data.setdefault('todos', {})
+
+ def clear_doc(self, docname: str) -> None:
+ self.todos.pop(docname, None)
+
+ def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> None:
+ for docname in docnames:
+ self.todos[docname] = otherdata['todos'][docname]
+
+ def process_doc(self, env: BuildEnvironment, docname: str,
+ document: nodes.document) -> None:
+ todos = self.todos.setdefault(docname, [])
+ for todo in document.findall(todo_node):
+ env.app.emit('todo-defined', todo)
+ todos.append(todo)
+
+ if env.config.todo_emit_warnings:
+ logger.warning(__("TODO entry found: %s"), todo[1].astext(),
+ location=todo)
+
class TodoList(SphinxDirective):
"""
A list of all todo entries.
"""
+
has_content = False
required_arguments = 0
optional_arguments = 0
final_argument_whitespace = False
option_spec: ClassVar[OptionSpec] = {}
+ def run(self) -> list[Node]:
+ # Simply insert an empty todolist node which will be replaced later
+ # when process_todo_nodes is called
+ return [todolist('')]
-class TodoListProcessor:
- def __init__(self, app: Sphinx, doctree: nodes.document, docname: str
- ) ->None:
+class TodoListProcessor:
+ def __init__(self, app: Sphinx, doctree: nodes.document, docname: str) -> None:
self.builder = app.builder
self.config = app.config
self.env = app.env
self.domain = cast(TodoDomain, app.env.get_domain('todo'))
self.document = new_document('')
+
self.process(doctree, docname)
- def resolve_reference(self, todo: todo_node, docname: str) ->None:
+ def process(self, doctree: nodes.document, docname: str) -> None:
+ todos: list[todo_node] = functools.reduce(
+ operator.iadd, self.domain.todos.values(), [])
+ for node in list(doctree.findall(todolist)):
+ if not self.config.todo_include_todos:
+ node.parent.remove(node)
+ continue
+
+ if node.get('ids'):
+ content: list[Element] = [nodes.target()]
+ else:
+ content = []
+
+ for todo in todos:
+ # Create a copy of the todo node
+ new_todo = todo.deepcopy()
+ new_todo['ids'].clear()
+
+ self.resolve_reference(new_todo, docname)
+ content.append(new_todo)
+
+ todo_ref = self.create_todo_reference(todo, docname)
+ content.append(todo_ref)
+
+ node.replace_self(content)
+
+ def create_todo_reference(self, todo: todo_node, docname: str) -> nodes.paragraph:
+ if self.config.todo_link_only:
+ description = _('<<original entry>>')
+ else:
+ description = (_('(The <<original entry>> is located in %s, line %d.)') %
+ (todo.source, todo.line))
+
+ prefix = description[:description.find('<<')]
+ suffix = description[description.find('>>') + 2:]
+
+ para = nodes.paragraph(classes=['todo-source'])
+ para += nodes.Text(prefix)
+
+ # Create a reference
+ linktext = nodes.emphasis(_('original entry'), _('original entry'))
+ reference = nodes.reference('', '', linktext, internal=True)
+ try:
+ reference['refuri'] = self.builder.get_relative_uri(docname, todo['docname'])
+ reference['refuri'] += '#' + todo['ids'][0]
+ except NoUri:
+ # ignore if no URI can be determined, e.g. for LaTeX output
+ pass
+
+ para += reference
+ para += nodes.Text(suffix)
+
+ return para
+
+ def resolve_reference(self, todo: todo_node, docname: str) -> None:
"""Resolve references in the todo content."""
- pass
+ for node in todo.findall(addnodes.pending_xref):
+ if 'refdoc' in node:
+ node['refdoc'] = docname
+
+ # Note: To resolve references, it is needed to wrap it with document node
+ self.document += todo
+ self.env.resolve_references(self.document, docname, self.builder)
+ self.document.remove(todo)
+
+
+def visit_todo_node(self: HTML5Translator, node: todo_node) -> None:
+ if self.config.todo_include_todos:
+ self.visit_admonition(node)
+ else:
+ raise nodes.SkipNode
+
+
+def depart_todo_node(self: HTML5Translator, node: todo_node) -> None:
+ self.depart_admonition(node)
+
+
+def latex_visit_todo_node(self: LaTeXTranslator, node: todo_node) -> None:
+ if self.config.todo_include_todos:
+ self.body.append('\n\\begin{sphinxtodo}{')
+ self.body.append(self.hypertarget_to(node))
+
+ title_node = cast(nodes.title, node[0])
+ title = texescape.escape(title_node.astext(), self.config.latex_engine)
+ self.body.append('%s:}' % title)
+ self.no_latex_floats += 1
+ if self.table:
+ self.table.has_problematic = True
+ node.pop(0)
+ else:
+ raise nodes.SkipNode
+
+
+def latex_depart_todo_node(self: LaTeXTranslator, node: todo_node) -> None:
+ self.body.append('\\end{sphinxtodo}\n')
+ self.no_latex_floats -= 1
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_event('todo-defined')
+ app.add_config_value('todo_include_todos', False, 'html')
+ app.add_config_value('todo_link_only', False, 'html')
+ app.add_config_value('todo_emit_warnings', False, 'html')
+
+ app.add_node(todolist)
+ app.add_node(todo_node,
+ html=(visit_todo_node, depart_todo_node),
+ latex=(latex_visit_todo_node, latex_depart_todo_node),
+ text=(visit_todo_node, depart_todo_node),
+ man=(visit_todo_node, depart_todo_node),
+ texinfo=(visit_todo_node, depart_todo_node))
+
+ app.add_directive('todo', Todo)
+ app.add_directive('todolist', TodoList)
+ app.add_domain(TodoDomain)
+ app.connect('doctree-resolved', TodoListProcessor)
+ return {
+ 'version': sphinx.__display_version__,
+ 'env_version': 2,
+ 'parallel_read_safe': True,
+ }
diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py
index 2fc49bef8..9991cf5d4 100644
--- a/sphinx/ext/viewcode.py
+++ b/sphinx/ext/viewcode.py
@@ -1,13 +1,17 @@
"""Add links to module code in Python object descriptions."""
+
from __future__ import annotations
+
import operator
import posixpath
import traceback
from importlib import import_module
from os import path
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes
from docutils.nodes import Element, Node
+
import sphinx
from sphinx import addnodes
from sphinx.builders.html import StandaloneHTMLBuilder
@@ -18,13 +22,18 @@ from sphinx.util import logging
from sphinx.util.display import status_iterator
from sphinx.util.nodes import make_refnode
from sphinx.util.osutil import _last_modified_time
+
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
+
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
+
+
OUTPUT_DIRNAME = '_modules'
@@ -37,16 +46,318 @@ class viewcode_anchor(Element):
"""
+def _get_full_modname(modname: str, attribute: str) -> str | None:
+ try:
+ if modname is None:
+ # Prevents a TypeError: if the last getattr() call will return None
+ # then it's better to return it directly
+ return None
+ module = import_module(modname)
+
+ # Allow an attribute to have multiple parts and incidentally allow
+ # repeated .s in the attribute.
+ value = module
+ for attr in attribute.split('.'):
+ if attr:
+ value = getattr(value, attr)
+
+ return getattr(value, '__module__', None)
+ except AttributeError:
+ # sphinx.ext.viewcode can't follow class instance attribute
+ # then AttributeError logging output only verbose mode.
+ logger.verbose("Didn't find %s in %s", attribute, modname)
+ return None
+ except Exception as e:
+ # sphinx.ext.viewcode follow python domain directives.
+ # because of that, if there are no real modules exists that specified
+ # by py:function or other directives, viewcode emits a lot of warnings.
+ # It should be displayed only verbose mode.
+ logger.verbose(traceback.format_exc().rstrip())
+ logger.verbose('viewcode can\'t import %s, failed with error "%s"', modname, e)
+ return None
+
+
+def is_supported_builder(builder: Builder) -> bool:
+ if builder.format != 'html':
+ return False
+ if builder.name == 'singlehtml':
+ return False
+ return not (builder.name.startswith('epub') and not builder.config.viewcode_enable_epub)
+
+
+def doctree_read(app: Sphinx, doctree: Node) -> None:
+ env = app.builder.env
+ if not hasattr(env, '_viewcode_modules'):
+ env._viewcode_modules = {} # type: ignore[attr-defined]
+
+ def has_tag(modname: str, fullname: str, docname: str, refname: str) -> bool:
+ entry = env._viewcode_modules.get(modname, None) # type: ignore[attr-defined]
+ if entry is False:
+ return False
+
+ code_tags = app.emit_firstresult('viewcode-find-source', modname)
+ if code_tags is None:
+ try:
+ analyzer = ModuleAnalyzer.for_module(modname)
+ analyzer.find_tags()
+ except Exception:
+ env._viewcode_modules[modname] = False # type: ignore[attr-defined]
+ return False
+
+ code = analyzer.code
+ tags = analyzer.tags
+ else:
+ code, tags = code_tags
+
+ if entry is None or entry[0] != code:
+ entry = code, tags, {}, refname
+ env._viewcode_modules[modname] = entry # type: ignore[attr-defined]
+ _, tags, used, _ = entry
+ if fullname in tags:
+ used[fullname] = docname
+ return True
+
+ return False
+
+ for objnode in list(doctree.findall(addnodes.desc)):
+ if objnode.get('domain') != 'py':
+ continue
+ names: set[str] = set()
+ for signode in objnode:
+ if not isinstance(signode, addnodes.desc_signature):
+ continue
+ modname = signode.get('module')
+ fullname = signode.get('fullname')
+ refname = modname
+ if env.config.viewcode_follow_imported_members:
+ new_modname = app.emit_firstresult(
+ 'viewcode-follow-imported', modname, fullname,
+ )
+ if not new_modname:
+ new_modname = _get_full_modname(modname, fullname)
+ modname = new_modname
+ if not modname:
+ continue
+ fullname = signode.get('fullname')
+ if not has_tag(modname, fullname, env.docname, refname):
+ continue
+ if fullname in names:
+ # only one link per name, please
+ continue
+ names.add(fullname)
+ pagename = posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/'))
+ signode += viewcode_anchor(reftarget=pagename, refid=fullname, refdoc=env.docname)
+
+
+def env_merge_info(app: Sphinx, env: BuildEnvironment, docnames: Iterable[str],
+ other: BuildEnvironment) -> None:
+ if not hasattr(other, '_viewcode_modules'):
+ return
+ # create a _viewcode_modules dict on the main environment
+ if not hasattr(env, '_viewcode_modules'):
+ env._viewcode_modules = {} # type: ignore[attr-defined]
+ # now merge in the information from the subprocess
+ for modname, entry in other._viewcode_modules.items():
+ if modname not in env._viewcode_modules: # type: ignore[attr-defined]
+ env._viewcode_modules[modname] = entry # type: ignore[attr-defined]
+ else:
+ if env._viewcode_modules[modname]: # type: ignore[attr-defined]
+ used = env._viewcode_modules[modname][2] # type: ignore[attr-defined]
+ for fullname, docname in entry[2].items():
+ if fullname not in used:
+ used[fullname] = docname
+
+
+def env_purge_doc(app: Sphinx, env: BuildEnvironment, docname: str) -> None:
+ modules = getattr(env, '_viewcode_modules', {})
+
+ for modname, entry in list(modules.items()):
+ if entry is False:
+ continue
+
+ code, tags, used, refname = entry
+ for fullname in list(used):
+ if used[fullname] == docname:
+ used.pop(fullname)
+
+ if len(used) == 0:
+ modules.pop(modname)
+
+
class ViewcodeAnchorTransform(SphinxPostTransform):
"""Convert or remove viewcode_anchor nodes depends on builder."""
+
default_priority = 100
+ def run(self, **kwargs: Any) -> None:
+ if is_supported_builder(self.app.builder):
+ self.convert_viewcode_anchors()
+ else:
+ self.remove_viewcode_anchors()
+
+ def convert_viewcode_anchors(self) -> None:
+ for node in self.document.findall(viewcode_anchor):
+ anchor = nodes.inline('', _('[source]'), classes=['viewcode-link'])
+ refnode = make_refnode(self.app.builder, node['refdoc'], node['reftarget'],
+ node['refid'], anchor)
+ node.replace_self(refnode)
+
+ def remove_viewcode_anchors(self) -> None:
+ for node in list(self.document.findall(viewcode_anchor)):
+ node.parent.remove(node)
-def get_module_filename(app: Sphinx, modname: str) ->(str | None):
+
+def get_module_filename(app: Sphinx, modname: str) -> str | None:
"""Get module filename for *modname*."""
- pass
+ source_info = app.emit_firstresult('viewcode-find-source', modname)
+ if source_info:
+ return None
+ else:
+ try:
+ filename, source = ModuleAnalyzer.get_module_source(modname)
+ return filename
+ except Exception:
+ return None
-def should_generate_module_page(app: Sphinx, modname: str) ->bool:
+def should_generate_module_page(app: Sphinx, modname: str) -> bool:
"""Check generation of module page is needed."""
- pass
+ module_filename = get_module_filename(app, modname)
+ if module_filename is None:
+ # Always (re-)generate module page when module filename is not found.
+ return True
+
+ builder = cast(StandaloneHTMLBuilder, app.builder)
+ basename = modname.replace('.', '/') + builder.out_suffix
+ page_filename = path.join(app.outdir, '_modules/', basename)
+
+ try:
+ if _last_modified_time(module_filename) <= _last_modified_time(page_filename):
+ # generation is not needed if the HTML page is newer than module file.
+ return False
+ except OSError:
+ pass
+
+ return True
+
+
+def collect_pages(app: Sphinx) -> Iterator[tuple[str, dict[str, Any], str]]:
+ env = app.builder.env
+ if not hasattr(env, '_viewcode_modules'):
+ return
+ if not is_supported_builder(app.builder):
+ return
+ highlighter = app.builder.highlighter # type: ignore[attr-defined]
+ urito = app.builder.get_relative_uri
+
+ modnames = set(env._viewcode_modules)
+
+ for modname, entry in status_iterator(
+ sorted(env._viewcode_modules.items()),
+ __('highlighting module code... '), "blue",
+ len(env._viewcode_modules),
+ app.verbosity, operator.itemgetter(0)):
+ if not entry:
+ continue
+ if not should_generate_module_page(app, modname):
+ continue
+
+ code, tags, used, refname = entry
+ # construct a page name for the highlighted source
+ pagename = posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/'))
+ # highlight the source using the builder's highlighter
+ if env.config.highlight_language in {'default', 'none'}:
+ lexer = env.config.highlight_language
+ else:
+ lexer = 'python'
+ linenos = 'inline' * env.config.viewcode_line_numbers
+ highlighted = highlighter.highlight_block(code, lexer, linenos=linenos)
+ # split the code into lines
+ lines = highlighted.splitlines()
+ # split off wrap markup from the first line of the actual code
+ before, after = lines[0].split('<pre>')
+ lines[0:1] = [before + '<pre>', after]
+ # nothing to do for the last line; it always starts with </pre> anyway
+ # now that we have code lines (starting at index 1), insert anchors for
+ # the collected tags (HACK: this only works if the tag boundaries are
+ # properly nested!)
+ max_index = len(lines) - 1
+ link_text = _('[docs]')
+ for name, docname in used.items():
+ type, start, end = tags[name]
+ backlink = urito(pagename, docname) + '#' + refname + '.' + name
+ lines[start] = (f'<div class="viewcode-block" id="{name}">\n'
+ f'<a class="viewcode-back" href="{backlink}">{link_text}</a>\n'
+ + lines[start])
+ lines[min(end, max_index)] += '</div>\n'
+
+ # try to find parents (for submodules)
+ parents = []
+ parent = modname
+ while '.' in parent:
+ parent = parent.rsplit('.', 1)[0]
+ if parent in modnames:
+ parents.append({
+ 'link': urito(pagename,
+ posixpath.join(OUTPUT_DIRNAME, parent.replace('.', '/'))),
+ 'title': parent})
+ parents.append({'link': urito(pagename, posixpath.join(OUTPUT_DIRNAME, 'index')),
+ 'title': _('Module code')})
+ parents.reverse()
+ # putting it all together
+ context = {
+ 'parents': parents,
+ 'title': modname,
+ 'body': (_('<h1>Source code for %s</h1>') % modname +
+ '\n'.join(lines)),
+ }
+ yield (pagename, context, 'page.html')
+
+ if not modnames:
+ return
+
+ html = ['\n']
+ # the stack logic is needed for using nested lists for submodules
+ stack = ['']
+ for modname in sorted(modnames):
+ if modname.startswith(stack[-1]):
+ stack.append(modname + '.')
+ html.append('<ul>')
+ else:
+ stack.pop()
+ while not modname.startswith(stack[-1]):
+ stack.pop()
+ html.append('</ul>')
+ stack.append(modname + '.')
+ relative_uri = urito(posixpath.join(OUTPUT_DIRNAME, 'index'),
+ posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/')))
+ html.append(f'<li><a href="{relative_uri}">{modname}</a></li>\n')
+ html.append('</ul>' * (len(stack) - 1))
+ context = {
+ 'title': _('Overview: module code'),
+ 'body': (_('<h1>All modules for which code is available</h1>') +
+ ''.join(html)),
+ }
+
+ yield (posixpath.join(OUTPUT_DIRNAME, 'index'), context, 'page.html')
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_config_value('viewcode_import', None, '')
+ app.add_config_value('viewcode_enable_epub', False, '')
+ app.add_config_value('viewcode_follow_imported_members', True, '')
+ app.add_config_value('viewcode_line_numbers', False, 'env', bool)
+ app.connect('doctree-read', doctree_read)
+ app.connect('env-merge-info', env_merge_info)
+ app.connect('env-purge-doc', env_purge_doc)
+ app.connect('html-collect-pages', collect_pages)
+ # app.add_config_value('viewcode_include_modules', [], 'env')
+ # app.add_config_value('viewcode_exclude_modules', [], 'env')
+ app.add_event('viewcode-find-source')
+ app.add_event('viewcode-follow-imported')
+ app.add_post_transform(ViewcodeAnchorTransform)
+ return {
+ 'version': sphinx.__display_version__,
+ 'env_version': 1,
+ 'parallel_read_safe': True,
+ }
diff --git a/sphinx/extension.py b/sphinx/extension.py
index 816225dad..88b9d420a 100644
--- a/sphinx/extension.py
+++ b/sphinx/extension.py
@@ -1,29 +1,42 @@
"""Utilities for Sphinx extensions."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any
+
from packaging.version import InvalidVersion, Version
+
from sphinx.errors import VersionRequirementError
from sphinx.locale import __
from sphinx.util import logging
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
class Extension:
-
- def __init__(self, name: str, module: Any, **kwargs: Any) ->None:
+ def __init__(self, name: str, module: Any, **kwargs: Any) -> None:
self.name = name
self.module = module
- self.metadata: ExtensionMetadata = kwargs
+ self.metadata: ExtensionMetadata = kwargs # type: ignore[assignment]
self.version = kwargs.pop('version', 'unknown version')
+
+ # The extension supports parallel read or not. The default value
+ # is ``None``. It means the extension does not tell the status.
+ # It will be warned on parallel reading.
self.parallel_read_safe = kwargs.pop('parallel_read_safe', None)
+
+ # The extension supports parallel write or not. The default value
+ # is ``True``. Sphinx writes parallelly documents even if
+ # the extension does not tell its status.
self.parallel_write_safe = kwargs.pop('parallel_write_safe', True)
-def verify_needs_extensions(app: Sphinx, config: Config) ->None:
+def verify_needs_extensions(app: Sphinx, config: Config) -> None:
"""Check that extensions mentioned in :confval:`needs_extensions` satisfy the version
requirement, and warn if an extension is not loaded.
@@ -32,4 +45,48 @@ def verify_needs_extensions(app: Sphinx, config: Config) ->None:
:raises VersionRequirementError: if the version of an extension in
:confval:`needs_extension` is unknown or older than the required version.
"""
- pass
+ if config.needs_extensions is None:
+ return
+
+ for extname, reqversion in config.needs_extensions.items():
+ extension = app.extensions.get(extname)
+ if extension is None:
+ logger.warning(
+ __(
+ 'The %s extension is required by needs_extensions settings, '
+ 'but it is not loaded.'
+ ),
+ extname,
+ )
+ continue
+
+ fulfilled = True
+ if extension.version == 'unknown version':
+ fulfilled = False
+ else:
+ try:
+ if Version(reqversion) > Version(extension.version):
+ fulfilled = False
+ except InvalidVersion:
+ if reqversion > extension.version:
+ fulfilled = False
+
+ if not fulfilled:
+ raise VersionRequirementError(
+ __(
+ 'This project needs the extension %s at least in '
+ 'version %s and therefore cannot be built with '
+ 'the loaded version (%s).'
+ )
+ % (extname, reqversion, extension.version)
+ )
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.connect('config-inited', verify_needs_extensions, priority=800)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py
index bef0da9d0..c7ae15668 100644
--- a/sphinx/highlighting.py
+++ b/sphinx/highlighting.py
@@ -1,69 +1,113 @@
"""Highlight code blocks using Pygments."""
+
from __future__ import annotations
+
from functools import partial
from importlib import import_module
from typing import TYPE_CHECKING, Any
+
import pygments
from pygments import highlight
from pygments.filters import ErrorToken
from pygments.formatters import HtmlFormatter, LatexFormatter
-from pygments.lexers import CLexer, PythonConsoleLexer, PythonLexer, RstLexer, TextLexer, get_lexer_by_name, guess_lexer
+from pygments.lexers import (
+ CLexer,
+ PythonConsoleLexer,
+ PythonLexer,
+ RstLexer,
+ TextLexer,
+ get_lexer_by_name,
+ guess_lexer,
+)
from pygments.styles import get_style_by_name
from pygments.util import ClassNotFound
+
from sphinx.locale import __
from sphinx.pygments_styles import NoneStyle, SphinxStyle
from sphinx.util import logging, texescape
+
if TYPE_CHECKING:
from pygments.formatter import Formatter
from pygments.lexer import Lexer
from pygments.style import Style
+
if tuple(map(int, pygments.__version__.split('.')))[:2] < (2, 18):
- from pygments.formatter import Formatter
- Formatter.__class_getitem__ = classmethod(lambda cls, name: cls)
+ from pygments.formatter import Formatter # NoQA: F811
+
+ Formatter.__class_getitem__ = classmethod(lambda cls, name: cls) # type: ignore[attr-defined]
+
logger = logging.getLogger(__name__)
+
lexers: dict[str, Lexer] = {}
-lexer_classes: dict[str, type[Lexer] | partial[Lexer]] = {'none': partial(
- TextLexer, stripnl=False), 'python': partial(PythonLexer, stripnl=False
- ), 'pycon': partial(PythonConsoleLexer, stripnl=False), 'rest': partial
- (RstLexer, stripnl=False), 'c': partial(CLexer, stripnl=False)}
-escape_hl_chars = {ord('\\'): '\\PYGZbs{}', ord('{'): '\\PYGZob{}', ord('}'
- ): '\\PYGZcb{}'}
-_LATEX_ADD_STYLES = """
+lexer_classes: dict[str, type[Lexer] | partial[Lexer]] = {
+ 'none': partial(TextLexer, stripnl=False),
+ 'python': partial(PythonLexer, stripnl=False),
+ 'pycon': partial(PythonConsoleLexer, stripnl=False),
+ 'rest': partial(RstLexer, stripnl=False),
+ 'c': partial(CLexer, stripnl=False),
+}
+
+
+escape_hl_chars = {
+ ord('\\'): '\\PYGZbs{}',
+ ord('{'): '\\PYGZob{}',
+ ord('}'): '\\PYGZcb{}',
+}
+
+# used if Pygments is available
+# MEMO: no use of \protected here to avoid having to do hyperref extras,
+# (if in future code highlighting in sectioning titles is activated):
+# the definitions here use only robust, protected or chardef tokens,
+# which are all known to the hyperref re-encoding for bookmarks.
+# The " is troublesome because we would like to use \text\textquotedbl
+# but \textquotedbl is *defined to raise an error* (!) if the font
+# encoding is OT1. This however could happen from 'fontenc' key.
+# MEMO: the Pygments escapes with \char`\<char> syntax, if the document
+# uses old OT1 font encoding, work correctly only in monospace font.
+# MEMO: the Pygmentize output mark-up is always with a {} after.
+_LATEX_ADD_STYLES = r"""
% Sphinx redefinitions
% Originally to obtain a straight single quote via package textcomp, then
% to fix problems for the 5.0.0 inline code highlighting (captions!).
-% The \\text is from amstext, a dependency of sphinx.sty. It is here only
+% The \text is from amstext, a dependency of sphinx.sty. It is here only
% to avoid build errors if for some reason expansion is in math mode.
-\\def\\PYGZbs{\\text\\textbackslash}
-\\def\\PYGZus{\\_}
-\\def\\PYGZob{\\{}
-\\def\\PYGZcb{\\}}
-\\def\\PYGZca{\\text\\textasciicircum}
-\\def\\PYGZam{\\&}
-\\def\\PYGZlt{\\text\\textless}
-\\def\\PYGZgt{\\text\\textgreater}
-\\def\\PYGZsh{\\#}
-\\def\\PYGZpc{\\%}
-\\def\\PYGZdl{\\$}
-\\def\\PYGZhy{\\sphinxhyphen}% defined in sphinxlatexstyletext.sty
-\\def\\PYGZsq{\\text\\textquotesingle}
-\\def\\PYGZdq{"}
-\\def\\PYGZti{\\text\\textasciitilde}
-\\makeatletter
-% use \\protected to allow syntax highlighting in captions
-\\protected\\def\\PYG#1#2{\\PYG@reset\\PYG@toks#1+\\relax+{\\PYG@do{#2}}}
-\\makeatother
+\def\PYGZbs{\text\textbackslash}
+\def\PYGZus{\_}
+\def\PYGZob{\{}
+\def\PYGZcb{\}}
+\def\PYGZca{\text\textasciicircum}
+\def\PYGZam{\&}
+\def\PYGZlt{\text\textless}
+\def\PYGZgt{\text\textgreater}
+\def\PYGZsh{\#}
+\def\PYGZpc{\%}
+\def\PYGZdl{\$}
+\def\PYGZhy{\sphinxhyphen}% defined in sphinxlatexstyletext.sty
+\def\PYGZsq{\text\textquotesingle}
+\def\PYGZdq{"}
+\def\PYGZti{\text\textasciitilde}
+\makeatletter
+% use \protected to allow syntax highlighting in captions
+\protected\def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+{\PYG@do{#2}}}
+\makeatother
"""
class PygmentsBridge:
+ # Set these attributes if you want to have different Pygments formatters
+ # than the default ones.
html_formatter = HtmlFormatter[str]
latex_formatter = LatexFormatter[str]
- def __init__(self, dest: str='html', stylename: str='sphinx',
- latex_engine: (str | None)=None) ->None:
+ def __init__(
+ self,
+ dest: str = 'html',
+ stylename: str = 'sphinx',
+ latex_engine: str | None = None,
+ ) -> None:
self.dest = dest
self.latex_engine = latex_engine
+
style = self.get_style(stylename)
self.formatter_args: dict[str, Any] = {'style': style}
if dest == 'html':
@@ -71,3 +115,117 @@ class PygmentsBridge:
else:
self.formatter = self.latex_formatter
self.formatter_args['commandprefix'] = 'PYG'
+
+ def get_style(self, stylename: str) -> type[Style]:
+ if not stylename or stylename == 'sphinx':
+ return SphinxStyle
+ elif stylename == 'none':
+ return NoneStyle
+ elif '.' in stylename:
+ module, stylename = stylename.rsplit('.', 1)
+ return getattr(import_module(module), stylename)
+ else:
+ return get_style_by_name(stylename)
+
+ def get_formatter(self, **kwargs: Any) -> Formatter:
+ kwargs.update(self.formatter_args)
+ return self.formatter(**kwargs)
+
+ def get_lexer(
+ self,
+ source: str,
+ lang: str,
+ opts: dict | None = None,
+ force: bool = False,
+ location: Any = None,
+ ) -> Lexer:
+ if not opts:
+ opts = {}
+
+ # find out which lexer to use
+ if lang in {'py', 'python', 'py3', 'python3', 'default'}:
+ if source.startswith('>>>'):
+ # interactive session
+ lang = 'pycon'
+ else:
+ lang = 'python'
+ if lang == 'pycon3':
+ lang = 'pycon'
+
+ if lang in lexers:
+ # just return custom lexers here (without installing raiseonerror filter)
+ return lexers[lang]
+ elif lang in lexer_classes:
+ lexer = lexer_classes[lang](**opts)
+ else:
+ try:
+ if lang == 'guess':
+ lexer = guess_lexer(source, **opts)
+ else:
+ lexer = get_lexer_by_name(lang, **opts)
+ except ClassNotFound:
+ logger.warning(
+ __('Pygments lexer name %r is not known'), lang, location=location
+ )
+ lexer = lexer_classes['none'](**opts)
+
+ if not force:
+ lexer.add_filter('raiseonerror')
+
+ return lexer
+
+ def highlight_block(
+ self,
+ source: str,
+ lang: str,
+ opts: dict | None = None,
+ force: bool = False,
+ location: Any = None,
+ **kwargs: Any,
+ ) -> str:
+ if not isinstance(source, str):
+ source = source.decode()
+
+ lexer = self.get_lexer(source, lang, opts, force, location)
+
+ # highlight via Pygments
+ formatter = self.get_formatter(**kwargs)
+ try:
+ hlsource = highlight(source, lexer, formatter)
+ except ErrorToken as err:
+ # this is most probably not the selected language,
+ # so let it pass un highlighted
+ if lang == 'default':
+ lang = 'none' # automatic highlighting failed.
+ else:
+ logger.warning(
+ __(
+ 'Lexing literal_block %r as "%s" resulted in an error at token: %r. '
+ 'Retrying in relaxed mode.'
+ ),
+ source,
+ lang,
+ str(err),
+ type='misc',
+ subtype='highlighting_failure',
+ location=location,
+ )
+ if force:
+ lang = 'none'
+ else:
+ force = True
+ lexer = self.get_lexer(source, lang, opts, force, location)
+ hlsource = highlight(source, lexer, formatter)
+
+ if self.dest == 'html':
+ return hlsource
+ else:
+ # MEMO: this is done to escape Unicode chars with non-Unicode engines
+ return texescape.hlescape(hlsource, self.latex_engine)
+
+ def get_stylesheet(self) -> str:
+ formatter = self.get_formatter()
+ if self.dest == 'html':
+ return formatter.get_style_defs('.highlight')
+ else:
+ return formatter.get_style_defs() + _LATEX_ADD_STYLES
diff --git a/sphinx/io.py b/sphinx/io.py
index 5f18053ac..31d64ca6d 100644
--- a/sphinx/io.py
+++ b/sphinx/io.py
@@ -1,50 +1,91 @@
"""Input/Output files"""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any
+
from docutils.core import Publisher
from docutils.io import FileInput, Input, NullOutput
from docutils.readers import standalone
from docutils.transforms.references import DanglingReferences
from docutils.writers import UnfilteredWriter
+
from sphinx import addnodes
from sphinx.transforms import AutoIndexUpgrader, DoctreeReadEvent, SphinxTransformer
-from sphinx.transforms.i18n import Locale, PreserveTranslatableMessages, RemoveTranslatableInline
+from sphinx.transforms.i18n import (
+ Locale,
+ PreserveTranslatableMessages,
+ RemoveTranslatableInline,
+)
from sphinx.transforms.references import SphinxDomains
from sphinx.util import logging
from sphinx.util.docutils import LoggingReporter
from sphinx.versioning import UIDTransform
+
if TYPE_CHECKING:
from docutils import nodes
from docutils.frontend import Values
from docutils.parsers import Parser
from docutils.transforms import Transform
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
+
+
logger = logging.getLogger(__name__)
-class SphinxBaseReader(standalone.Reader):
+class SphinxBaseReader(standalone.Reader): # type: ignore[misc]
"""
A base class of readers for Sphinx.
This replaces reporter by Sphinx's on generating document.
"""
+
transforms: list[type[Transform]] = []
- def __init__(self, *args: Any, **kwargs: Any) ->None:
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
from sphinx.application import Sphinx
+
if len(args) > 0 and isinstance(args[0], Sphinx):
self._app = args[0]
self._env = self._app.env
args = args[1:]
+
super().__init__(*args, **kwargs)
- def new_document(self) ->nodes.document:
+ def setup(self, app: Sphinx) -> None:
+ self._app = app # hold application object only for compatibility
+ self._env = app.env
+
+ def get_transforms(self) -> list[type[Transform]]:
+ transforms = super().get_transforms() + self.transforms
+
+ # remove transforms which is not needed for Sphinx
+ unused = [DanglingReferences]
+ for transform in unused:
+ if transform in transforms:
+ transforms.remove(transform)
+
+ return transforms
+
+ def new_document(self) -> nodes.document:
"""
Creates a new document object which has a special reporter object good
for logging.
"""
- pass
+ document = super().new_document()
+ document.__class__ = addnodes.document # replace the class with patched version
+
+ # substitute transformer
+ document.transformer = SphinxTransformer(document)
+ document.transformer.set_environment(self.settings.env)
+
+ # substitute reporter
+ reporter = document.reporter
+ document.reporter = LoggingReporter.from_reporter(reporter)
+
+ return document
class SphinxStandaloneReader(SphinxBaseReader):
@@ -52,9 +93,27 @@ class SphinxStandaloneReader(SphinxBaseReader):
A basic document reader for Sphinx.
"""
- def read_source(self, env: BuildEnvironment) ->str:
+ def setup(self, app: Sphinx) -> None:
+ self.transforms = self.transforms + app.registry.get_transforms()
+ super().setup(app)
+
+ def read(self, source: Input, parser: Parser, settings: Values) -> nodes.document: # type: ignore[type-arg]
+ self.source = source
+ if not self.parser: # type: ignore[has-type]
+ self.parser = parser
+ self.settings = settings
+ self.input = self.read_source(settings.env)
+ self.parse()
+ return self.document
+
+ def read_source(self, env: BuildEnvironment) -> str:
"""Read content from source and do post-process."""
- pass
+ content = self.source.read()
+
+ # emit "source-read" event
+ arg = [content]
+ env.events.emit('source-read', env.docname, arg)
+ return arg[0]
class SphinxI18nReader(SphinxBaseReader):
@@ -66,20 +125,70 @@ class SphinxI18nReader(SphinxBaseReader):
Because the translated texts are partial and they don't have correct line numbers.
"""
-
-class SphinxDummyWriter(UnfilteredWriter):
+ def setup(self, app: Sphinx) -> None:
+ super().setup(app)
+
+ self.transforms = self.transforms + app.registry.get_transforms()
+ unused = [
+ PreserveTranslatableMessages,
+ Locale,
+ RemoveTranslatableInline,
+ AutoIndexUpgrader,
+ SphinxDomains,
+ DoctreeReadEvent,
+ UIDTransform,
+ ]
+ for transform in unused:
+ if transform in self.transforms:
+ self.transforms.remove(transform)
+
+
+class SphinxDummyWriter(UnfilteredWriter): # type: ignore[misc]
"""Dummy writer module used for generating doctree."""
- supported = 'html',
+ supported = ('html',) # needed to keep "meta" nodes
+
+ def translate(self) -> None:
+ pass
-def SphinxDummySourceClass(source: Any, *args: Any, **kwargs: Any) ->Any:
+
+def SphinxDummySourceClass(source: Any, *args: Any, **kwargs: Any) -> Any:
"""Bypass source object as is to cheat Publisher."""
- pass
+ return source
class SphinxFileInput(FileInput):
"""A basic FileInput for Sphinx."""
- def __init__(self, *args: Any, **kwargs: Any) ->None:
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
kwargs['error_handler'] = 'sphinx'
super().__init__(*args, **kwargs)
+
+
+def create_publisher(app: Sphinx, filetype: str) -> Publisher:
+ reader = SphinxStandaloneReader()
+ reader.setup(app)
+
+ parser = app.registry.create_source_parser(app, filetype)
+ if parser.__class__.__name__ == 'CommonMarkParser' and parser.settings_spec == ():
+ # a workaround for recommonmark
+ # If recommonmark.AutoStrictify is enabled, the parser invokes reST parser
+ # internally. But recommonmark-0.4.0 does not provide settings_spec for reST
+ # parser. As a workaround, this copies settings_spec for RSTParser to the
+ # CommonMarkParser.
+ from docutils.parsers.rst import Parser as RSTParser
+
+ parser.settings_spec = RSTParser.settings_spec # type: ignore[misc]
+
+ pub = Publisher(
+ reader=reader,
+ parser=parser,
+ writer=SphinxDummyWriter(),
+ source_class=SphinxFileInput,
+ destination=NullOutput(),
+ )
+ # Propagate exceptions by default when used programmatically:
+ defaults = {'traceback': True, **app.env.settings}
+ # Set default settings
+ pub.get_settings(**defaults)
+ return pub
diff --git a/sphinx/jinja2glue.py b/sphinx/jinja2glue.py
index 6b3cabe75..0df58b52d 100644
--- a/sphinx/jinja2glue.py
+++ b/sphinx/jinja2glue.py
@@ -1,23 +1,43 @@
"""Glue code for the jinja2 templating engine."""
+
from __future__ import annotations
+
import os
from os import path
from pprint import pformat
from typing import TYPE_CHECKING, Any
+
from jinja2 import BaseLoader, FileSystemLoader, TemplateNotFound
from jinja2.sandbox import SandboxedEnvironment
from jinja2.utils import open_if_exists, pass_context
+
from sphinx.application import TemplateBridge
from sphinx.util import logging
from sphinx.util.osutil import _last_modified_time
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
+
from jinja2.environment import Environment
+
from sphinx.builders import Builder
from sphinx.theming import Theme
-def _todim(val: (int | str)) ->str:
+def _tobool(val: str) -> bool:
+ if isinstance(val, str):
+ return val.lower() in ('true', '1', 'yes', 'on')
+ return bool(val)
+
+
+def _toint(val: str) -> int:
+ try:
+ return int(val)
+ except ValueError:
+ return 0
+
+
+def _todim(val: int | str) -> str:
"""
Make val a css dimension. In particular the following transformations
are performed:
@@ -28,23 +48,66 @@ def _todim(val: (int | str)) ->str:
Everything else is returned unchanged.
"""
- pass
+ if val is None:
+ return 'initial'
+ elif str(val).isdigit():
+ return '0' if int(val) == 0 else '%spx' % val
+ return val # type: ignore[return-value]
+
+
+def _slice_index(values: list, slices: int) -> Iterator[list]:
+ seq = values.copy()
+ length = 0
+ for value in values:
+ length += 1 + len(value[1][1]) # count includes subitems
+ items_per_slice = length // slices
+ offset = 0
+ for slice_number in range(slices):
+ count = 0
+ start = offset
+ if slices == slice_number + 1: # last column
+ offset = len(seq) # NoQA: SIM113
+ else:
+ for value in values[offset:]:
+ count += 1 + len(value[1][1])
+ offset += 1
+ if count >= items_per_slice:
+ break
+ yield seq[start:offset]
-def accesskey(context: Any, key: str) ->str:
+def accesskey(context: Any, key: str) -> str:
"""Helper to output each access key only once."""
- pass
+ if '_accesskeys' not in context:
+ context.vars['_accesskeys'] = {}
+ if key and key not in context.vars['_accesskeys']:
+ context.vars['_accesskeys'][key] = 1
+ return 'accesskey="%s"' % key
+ return ''
class idgen:
-
- def __init__(self) ->None:
+ def __init__(self) -> None:
self.id = 0
- def __next__(self) ->int:
+ def current(self) -> int:
+ return self.id
+
+ def __next__(self) -> int:
self.id += 1
return self.id
- next = __next__
+
+ next = __next__ # Python 2/Jinja compatibility
+
+
+@pass_context
+def warning(context: dict, message: str, *args: Any, **kwargs: Any) -> str:
+ if 'pagename' in context:
+ filename = context.get('pagename') + context.get('file_suffix', '')
+ message = f'in rendering {filename}: {message}'
+ logger = logging.getLogger('sphinx.themes')
+ logger.warning(message, *args, **kwargs)
+ return '' # return empty string not to output any values
class SphinxFileSystemLoader(FileSystemLoader):
@@ -53,8 +116,124 @@ class SphinxFileSystemLoader(FileSystemLoader):
template names.
"""
+ def get_source(
+ self, environment: Environment, template: str
+ ) -> tuple[str, str, Callable]:
+ for searchpath in self.searchpath:
+ filename = path.join(searchpath, template)
+ f = open_if_exists(filename)
+ if f is not None:
+ break
+ else:
+ raise TemplateNotFound(template)
+
+ with f:
+ contents = f.read().decode(self.encoding)
+
+ mtime = _last_modified_time(filename)
+
+ def uptodate() -> bool:
+ try:
+ return _last_modified_time(filename) == mtime
+ except OSError:
+ return False
+
+ return contents, filename, uptodate
+
class BuiltinTemplateLoader(TemplateBridge, BaseLoader):
"""
Interfaces the rendering environment of jinja2 for use in Sphinx.
"""
+
+ # TemplateBridge interface
+
+ def init(
+ self,
+ builder: Builder,
+ theme: Theme | None = None,
+ dirs: list[str] | None = None,
+ ) -> None:
+ # create a chain of paths to search
+ if theme:
+ # the theme's own dir and its bases' dirs
+ pathchain = theme.get_theme_dirs()
+ # the loader dirs: pathchain + the parent directories for all themes
+ loaderchain = pathchain + [path.join(p, '..') for p in pathchain]
+ elif dirs:
+ pathchain = list(dirs)
+ loaderchain = list(dirs)
+ else:
+ pathchain = []
+ loaderchain = []
+
+ # prepend explicit template paths
+ self.templatepathlen = len(builder.config.templates_path)
+ if builder.config.templates_path:
+ cfg_templates_path = [
+ path.join(builder.confdir, tp) for tp in builder.config.templates_path
+ ]
+ pathchain[0:0] = cfg_templates_path
+ loaderchain[0:0] = cfg_templates_path
+
+ # store it for use in newest_template_mtime
+ self.pathchain = pathchain
+
+ # make the paths into loaders
+ self.loaders = [SphinxFileSystemLoader(x) for x in loaderchain]
+
+ use_i18n = builder.app.translator is not None
+ extensions = ['jinja2.ext.i18n'] if use_i18n else []
+ self.environment = SandboxedEnvironment(loader=self, extensions=extensions)
+ self.environment.filters['tobool'] = _tobool
+ self.environment.filters['toint'] = _toint
+ self.environment.filters['todim'] = _todim
+ self.environment.filters['slice_index'] = _slice_index
+ self.environment.globals['debug'] = pass_context(pformat)
+ self.environment.globals['warning'] = warning
+ self.environment.globals['accesskey'] = pass_context(accesskey)
+ self.environment.globals['idgen'] = idgen
+ if use_i18n:
+ # ``install_gettext_translations`` is injected by the ``jinja2.ext.i18n`` extension
+ self.environment.install_gettext_translations( # type: ignore[attr-defined]
+ builder.app.translator
+ )
+
+ def render(self, template: str, context: dict) -> str: # type: ignore[override]
+ return self.environment.get_template(template).render(context)
+
+ def render_string(self, source: str, context: dict) -> str:
+ return self.environment.from_string(source).render(context)
+
+ def newest_template_mtime(self) -> float:
+ return self._newest_template_mtime_name()[0]
+
+ def newest_template_name(self) -> str:
+ return self._newest_template_mtime_name()[1]
+
+ def _newest_template_mtime_name(self) -> tuple[float, str]:
+ return max(
+ (os.stat(os.path.join(root, sfile)).st_mtime_ns / 10**9, sfile)
+ for dirname in self.pathchain
+ for root, _dirs, files in os.walk(dirname)
+ for sfile in files
+ if sfile.endswith('.html')
+ )
+
+ # Loader interface
+
+ def get_source(
+ self, environment: Environment, template: str
+ ) -> tuple[str, str, Callable]:
+ loaders = self.loaders
+ # exclamation mark starts search from theme
+ if template.startswith('!'):
+ loaders = loaders[self.templatepathlen :]
+ template = template[1:]
+ for loader in loaders:
+ try:
+ return loader.get_source(environment, template)
+ except TemplateNotFound:
+ pass
+ msg = f'{template!r} not found in {self.environment.loader.pathchain}' # type: ignore[union-attr]
+ raise TemplateNotFound(msg)
diff --git a/sphinx/parsers.py b/sphinx/parsers.py
index a9686d339..cc10ce184 100644
--- a/sphinx/parsers.py
+++ b/sphinx/parsers.py
@@ -1,15 +1,21 @@
"""A Base class for additional parsers."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING
+
import docutils.parsers
import docutils.parsers.rst
from docutils import nodes
from docutils.parsers.rst import states
from docutils.statemachine import StringList
from docutils.transforms.universal import SmartQuotes
+
from sphinx.util.rst import append_epilog, prepend_prolog
+
if TYPE_CHECKING:
from docutils.transforms import Transform
+
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.environment import BuildEnvironment
@@ -24,33 +30,72 @@ class Parser(docutils.parsers.Parser):
The subclasses can access sphinx core runtime objects (app, config and env).
"""
+
+ #: The config object
config: Config
+
+ #: The environment object
env: BuildEnvironment
- def set_application(self, app: Sphinx) ->None:
+ def set_application(self, app: Sphinx) -> None:
"""set_application will be called from Sphinx to set app and other instance variables
:param sphinx.application.Sphinx app: Sphinx application object
"""
- pass
+ self._app = app
+ self.config = app.config
+ self.env = app.env
class RSTParser(docutils.parsers.rst.Parser, Parser):
"""A reST parser for Sphinx."""
- def get_transforms(self) ->list[type[Transform]]:
+ def get_transforms(self) -> list[type[Transform]]:
"""
Sphinx's reST parser replaces a transform class for smart-quotes by its own
refs: sphinx.io.SphinxStandaloneReader
"""
- pass
+ transforms = super().get_transforms()
+ transforms.remove(SmartQuotes)
+ return transforms
- def parse(self, inputstring: (str | StringList), document: nodes.document
- ) ->None:
+ def parse(self, inputstring: str | StringList, document: nodes.document) -> None:
"""Parse text and generate a document tree."""
- pass
+ self.setup_parse(inputstring, document) # type: ignore[arg-type]
+ self.statemachine = states.RSTStateMachine(
+ state_classes=self.state_classes,
+ initial_state=self.initial_state,
+ debug=document.reporter.debug_flag,
+ )
+
+ # preprocess inputstring
+ if isinstance(inputstring, str):
+ lines = docutils.statemachine.string2lines(
+ inputstring,
+ tab_width=document.settings.tab_width,
+ convert_whitespace=True,
+ )
+
+ inputlines = StringList(lines, document.current_source)
+ else:
+ inputlines = inputstring
+
+ self.decorate(inputlines)
+ self.statemachine.run(inputlines, document, inliner=self.inliner)
+ self.finish_parse()
- def decorate(self, content: StringList) ->None:
+ def decorate(self, content: StringList) -> None:
"""Preprocess reST content before parsing."""
- pass
+ prepend_prolog(content, self.config.rst_prolog)
+ append_epilog(content, self.config.rst_epilog)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_source_parser(RSTParser)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/project.py b/sphinx/project.py
index 7075b88a1..016429993 100644
--- a/sphinx/project.py
+++ b/sphinx/project.py
@@ -1,16 +1,21 @@
"""Utility function and classes for Sphinx projects."""
+
from __future__ import annotations
+
import contextlib
import os
from pathlib import Path
from typing import TYPE_CHECKING
+
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util._pathlib import _StrPath
from sphinx.util.matching import get_matching_files
from sphinx.util.osutil import path_stabilize
+
if TYPE_CHECKING:
from collections.abc import Iterable
+
logger = logging.getLogger(__name__)
EXCLUDE_PATHS = ['**/_sources', '.#*', '**/.#*', '*.lproj/**']
@@ -18,37 +23,106 @@ EXCLUDE_PATHS = ['**/_sources', '.#*', '**/.#*', '*.lproj/**']
class Project:
"""A project is the source code set of the Sphinx document(s)."""
- def __init__(self, srcdir: (str | os.PathLike[str]), source_suffix:
- Iterable[str]) ->None:
+ def __init__(
+ self, srcdir: str | os.PathLike[str], source_suffix: Iterable[str]
+ ) -> None:
+ #: Source directory.
self.srcdir = _StrPath(srcdir)
+
+ #: source_suffix. Same as :confval:`source_suffix`.
self.source_suffix = tuple(source_suffix)
self._first_source_suffix = next(iter(self.source_suffix), '')
+
+ #: The name of documents belonging to this project.
self.docnames: set[str] = set()
+
+ # Bijective mapping between docnames and (srcdir relative) paths.
self._path_to_docname: dict[Path, str] = {}
self._docname_to_path: dict[str, Path] = {}
- def restore(self, other: Project) ->None:
+ def restore(self, other: Project) -> None:
"""Take over a result of last build."""
- pass
+ self.docnames = other.docnames
+ self._path_to_docname = other._path_to_docname
+ self._docname_to_path = other._docname_to_path
- def discover(self, exclude_paths: Iterable[str]=(), include_paths:
- Iterable[str]=('**',)) ->set[str]:
+ def discover(
+ self, exclude_paths: Iterable[str] = (), include_paths: Iterable[str] = ('**',)
+ ) -> set[str]:
"""Find all document files in the source directory and put them in
:attr:`docnames`.
"""
- pass
+ self.docnames.clear()
+ self._path_to_docname.clear()
+ self._docname_to_path.clear()
+
+ for filename in get_matching_files(
+ self.srcdir,
+ include_paths,
+ [*exclude_paths, *EXCLUDE_PATHS],
+ ):
+ if docname := self.path2doc(filename):
+ if docname in self.docnames:
+ files = [
+ str(f.relative_to(self.srcdir))
+ for f in self.srcdir.glob(f'{docname}.*')
+ ]
+ logger.warning(
+ __(
+ 'multiple files found for the document "%s": %s\n'
+ 'Use %r for the build.'
+ ),
+ docname,
+ ', '.join(files),
+ self.doc2path(docname, absolute=True),
+ once=True,
+ )
+ elif os.access(self.srcdir / filename, os.R_OK):
+ self.docnames.add(docname)
+ path = Path(filename)
+ self._path_to_docname[path] = docname
+ self._docname_to_path[docname] = path
+ else:
+ logger.warning(
+ __('Ignored unreadable document %r.'),
+ filename,
+ location=docname,
+ )
+
+ return self.docnames
- def path2doc(self, filename: (str | os.PathLike[str])) ->(str | None):
+ def path2doc(self, filename: str | os.PathLike[str]) -> str | None:
"""Return the docname for the filename if the file is a document.
*filename* should be absolute or relative to the source directory.
"""
- pass
+ try:
+ return self._path_to_docname[filename] # type: ignore[index]
+ except KeyError:
+ path = Path(filename)
+ if path.is_absolute():
+ with contextlib.suppress(ValueError):
+ path = path.relative_to(self.srcdir)
- def doc2path(self, docname: str, absolute: bool) ->_StrPath:
+ for suffix in self.source_suffix:
+ if path.name.endswith(suffix):
+ return path_stabilize(path).removesuffix(suffix)
+
+ # the file does not have a docname
+ return None
+
+ def doc2path(self, docname: str, absolute: bool) -> _StrPath:
"""Return the filename for the document name.
If *absolute* is True, return as an absolute path.
Else, return as a relative path to the source directory.
"""
- pass
+ try:
+ filename = self._docname_to_path[docname]
+ except KeyError:
+ # Backwards compatibility: the document does not exist
+ filename = Path(docname + self._first_source_suffix)
+
+ if absolute:
+ return _StrPath(self.srcdir / filename)
+ return _StrPath(filename)
diff --git a/sphinx/pycode/ast.py b/sphinx/pycode/ast.py
index f7da51a16..7ed107f4a 100644
--- a/sphinx/pycode/ast.py
+++ b/sphinx/pycode/ast.py
@@ -1,27 +1,203 @@
"""Helpers for AST (Abstract Syntax Tree)."""
+
from __future__ import annotations
+
import ast
from typing import NoReturn, overload
-OPERATORS: dict[type[ast.AST], str] = {ast.Add: '+', ast.And: 'and', ast.
- BitAnd: '&', ast.BitOr: '|', ast.BitXor: '^', ast.Div: '/', ast.
- FloorDiv: '//', ast.Invert: '~', ast.LShift: '<<', ast.MatMult: '@',
- ast.Mult: '*', ast.Mod: '%', ast.Not: 'not', ast.Pow: '**', ast.Or:
- 'or', ast.RShift: '>>', ast.Sub: '-', ast.UAdd: '+', ast.USub: '-'}
+
+OPERATORS: dict[type[ast.AST], str] = {
+ ast.Add: "+",
+ ast.And: "and",
+ ast.BitAnd: "&",
+ ast.BitOr: "|",
+ ast.BitXor: "^",
+ ast.Div: "/",
+ ast.FloorDiv: "//",
+ ast.Invert: "~",
+ ast.LShift: "<<",
+ ast.MatMult: "@",
+ ast.Mult: "*",
+ ast.Mod: "%",
+ ast.Not: "not",
+ ast.Pow: "**",
+ ast.Or: "or",
+ ast.RShift: ">>",
+ ast.Sub: "-",
+ ast.UAdd: "+",
+ ast.USub: "-",
+}
+
+
+@overload
+def unparse(node: None, code: str = '') -> None:
+ ...
-def unparse(node: (ast.AST | None), code: str='') ->(str | None):
+@overload
+def unparse(node: ast.AST, code: str = '') -> str:
+ ...
+
+
+def unparse(node: ast.AST | None, code: str = '') -> str | None:
"""Unparse an AST to string."""
- pass
+ if node is None:
+ return None
+ elif isinstance(node, str):
+ return node
+ return _UnparseVisitor(code).visit(node)
+# a greatly cut-down version of `ast._Unparser`
class _UnparseVisitor(ast.NodeVisitor):
-
- def __init__(self, code: str='') ->None:
+ def __init__(self, code: str = '') -> None:
self.code = code
+
+ def _visit_op(self, node: ast.AST) -> str:
+ return OPERATORS[node.__class__]
for _op in OPERATORS:
locals()[f'visit_{_op.__name__}'] = _visit_op
- def _visit_arg_with_default(self, arg: ast.arg, default: (ast.AST | None)
- ) ->str:
+ def visit_arg(self, node: ast.arg) -> str:
+ if node.annotation:
+ return f"{node.arg}: {self.visit(node.annotation)}"
+ else:
+ return node.arg
+
+ def _visit_arg_with_default(self, arg: ast.arg, default: ast.AST | None) -> str:
"""Unparse a single argument to a string."""
- pass
+ name = self.visit(arg)
+ if default:
+ if arg.annotation:
+ name += " = %s" % self.visit(default)
+ else:
+ name += "=%s" % self.visit(default)
+ return name
+
+ def visit_arguments(self, node: ast.arguments) -> str:
+ defaults: list[ast.expr | None] = list(node.defaults)
+ positionals = len(node.args)
+ posonlyargs = len(node.posonlyargs)
+ positionals += posonlyargs
+ for _ in range(len(defaults), positionals):
+ defaults.insert(0, None)
+
+ kw_defaults: list[ast.expr | None] = list(node.kw_defaults)
+ for _ in range(len(kw_defaults), len(node.kwonlyargs)):
+ kw_defaults.insert(0, None)
+
+ args: list[str] = [self._visit_arg_with_default(arg, defaults[i])
+ for i, arg in enumerate(node.posonlyargs)]
+
+ if node.posonlyargs:
+ args.append('/')
+
+ for i, arg in enumerate(node.args):
+ args.append(self._visit_arg_with_default(arg, defaults[i + posonlyargs]))
+
+ if node.vararg:
+ args.append("*" + self.visit(node.vararg))
+
+ if node.kwonlyargs and not node.vararg:
+ args.append('*')
+ for i, arg in enumerate(node.kwonlyargs):
+ args.append(self._visit_arg_with_default(arg, kw_defaults[i]))
+
+ if node.kwarg:
+ args.append("**" + self.visit(node.kwarg))
+
+ return ", ".join(args)
+
+ def visit_Attribute(self, node: ast.Attribute) -> str:
+ return f"{self.visit(node.value)}.{node.attr}"
+
+ def visit_BinOp(self, node: ast.BinOp) -> str:
+ # Special case ``**`` to not have surrounding spaces.
+ if isinstance(node.op, ast.Pow):
+ return "".join(map(self.visit, (node.left, node.op, node.right)))
+ return " ".join(map(self.visit, (node.left, node.op, node.right)))
+
+ def visit_BoolOp(self, node: ast.BoolOp) -> str:
+ op = " %s " % self.visit(node.op)
+ return op.join(self.visit(e) for e in node.values)
+
+ def visit_Call(self, node: ast.Call) -> str:
+ args = ', '.join(
+ [self.visit(e) for e in node.args]
+ + [f"{k.arg}={self.visit(k.value)}" for k in node.keywords],
+ )
+ return f"{self.visit(node.func)}({args})"
+
+ def visit_Constant(self, node: ast.Constant) -> str:
+ if node.value is Ellipsis:
+ return "..."
+ elif isinstance(node.value, int | float | complex):
+ if self.code:
+ return ast.get_source_segment(self.code, node) or repr(node.value)
+ else:
+ return repr(node.value)
+ else:
+ return repr(node.value)
+
+ def visit_Dict(self, node: ast.Dict) -> str:
+ keys = (self.visit(k) for k in node.keys if k is not None)
+ values = (self.visit(v) for v in node.values)
+ items = (k + ": " + v for k, v in zip(keys, values, strict=True))
+ return "{" + ", ".join(items) + "}"
+
+ def visit_Lambda(self, node: ast.Lambda) -> str:
+ return "lambda %s: ..." % self.visit(node.args)
+
+ def visit_List(self, node: ast.List) -> str:
+ return "[" + ", ".join(self.visit(e) for e in node.elts) + "]"
+
+ def visit_Name(self, node: ast.Name) -> str:
+ return node.id
+
+ def visit_Set(self, node: ast.Set) -> str:
+ return "{" + ", ".join(self.visit(e) for e in node.elts) + "}"
+
+ def visit_Slice(self, node: ast.Slice) -> str:
+ if not node.lower and not node.upper and not node.step:
+ # Empty slice with default values -> [:]
+ return ":"
+
+ start = self.visit(node.lower) if node.lower else ""
+ stop = self.visit(node.upper) if node.upper else ""
+ if not node.step:
+ # Default step size -> [start:stop]
+ return f"{start}:{stop}"
+
+ step = self.visit(node.step) if node.step else ""
+ return f"{start}:{stop}:{step}"
+
+ def visit_Subscript(self, node: ast.Subscript) -> str:
+ def is_simple_tuple(value: ast.expr) -> bool:
+ return (
+ isinstance(value, ast.Tuple)
+ and bool(value.elts)
+ and not any(isinstance(elt, ast.Starred) for elt in value.elts)
+ )
+
+ if is_simple_tuple(node.slice):
+ elts = ", ".join(self.visit(e)
+ for e in node.slice.elts) # type: ignore[attr-defined]
+ return f"{self.visit(node.value)}[{elts}]"
+ return f"{self.visit(node.value)}[{self.visit(node.slice)}]"
+
+ def visit_UnaryOp(self, node: ast.UnaryOp) -> str:
+ # UnaryOp is one of {UAdd, USub, Invert, Not}, which refer to ``+x``,
+ # ``-x``, ``~x``, and ``not x``. Only Not needs a space.
+ if isinstance(node.op, ast.Not):
+ return f"{self.visit(node.op)} {self.visit(node.operand)}"
+ return f"{self.visit(node.op)}{self.visit(node.operand)}"
+
+ def visit_Tuple(self, node: ast.Tuple) -> str:
+ if len(node.elts) == 0:
+ return "()"
+ elif len(node.elts) == 1:
+ return "(%s,)" % self.visit(node.elts[0])
+ else:
+ return "(" + ", ".join(self.visit(e) for e in node.elts) + ")"
+
+ def generic_visit(self, node: ast.AST) -> NoReturn:
+ raise NotImplementedError('Unable to parse %s object' % type(node).__name__)
diff --git a/sphinx/pycode/parser.py b/sphinx/pycode/parser.py
index 54e6b77c5..18bb99305 100644
--- a/sphinx/pycode/parser.py
+++ b/sphinx/pycode/parser.py
@@ -1,5 +1,7 @@
"""Utilities parsing and analyzing Python code."""
+
from __future__ import annotations
+
import ast
import contextlib
import functools
@@ -12,18 +14,27 @@ from inspect import Signature
from token import DEDENT, INDENT, NAME, NEWLINE, NUMBER, OP, STRING
from tokenize import COMMENT, NL
from typing import Any
+
from sphinx.pycode.ast import unparse as ast_unparse
+
comment_re = re.compile('^\\s*#: ?(.*)\r?\n?$')
indent_re = re.compile('^\\s*$')
emptyline_re = re.compile('^\\s*(#.*)?$')
-def get_assign_targets(node: ast.AST) ->list[ast.expr]:
+def filter_whitespace(code: str) -> str:
+ return code.replace('\f', ' ') # replace FF (form feed) with whitespace
+
+
+def get_assign_targets(node: ast.AST) -> list[ast.expr]:
"""Get list of targets from Assign and AnnAssign node."""
- pass
+ if isinstance(node, ast.Assign):
+ return node.targets
+ else:
+ return [node.target] # type: ignore[attr-defined]
-def get_lvar_names(node: ast.AST, self: (ast.arg | None)=None) ->list[str]:
+def get_lvar_names(node: ast.AST, self: ast.arg | None = None) -> list[str]:
"""Convert assignment-AST to variable names.
This raises `TypeError` if the assignment does not create new variable::
@@ -32,26 +43,67 @@ def get_lvar_names(node: ast.AST, self: (ast.arg | None)=None) ->list[str]:
dic["bar"] = 'baz'
# => TypeError
"""
- pass
+ if self:
+ self_id = self.arg
+
+ node_name = node.__class__.__name__
+ if node_name in ('Constant', 'Index', 'Slice', 'Subscript'):
+ raise TypeError('%r does not create new variable' % node)
+ if node_name == 'Name':
+ if self is None or node.id == self_id: # type: ignore[attr-defined]
+ return [node.id] # type: ignore[attr-defined]
+ else:
+ raise TypeError('The assignment %r is not instance variable' % node)
+ elif node_name in ('Tuple', 'List'):
+ members = []
+ for elt in node.elts: # type: ignore[attr-defined]
+ with contextlib.suppress(TypeError):
+ members.extend(get_lvar_names(elt, self))
+
+ return members
+ elif node_name == 'Attribute':
+ if (
+ node.value.__class__.__name__ == 'Name' and # type: ignore[attr-defined]
+ self and node.value.id == self_id # type: ignore[attr-defined]
+ ):
+ # instance variable
+ return ["%s" % get_lvar_names(node.attr, self)[0]] # type: ignore[attr-defined]
+ else:
+ raise TypeError('The assignment %r is not instance variable' % node)
+ elif node_name == 'str':
+ return [node] # type: ignore[list-item]
+ elif node_name == 'Starred':
+ return get_lvar_names(node.value, self) # type: ignore[attr-defined]
+ else:
+ raise NotImplementedError('Unexpected node name %r' % node_name)
-def dedent_docstring(s: str) ->str:
+def dedent_docstring(s: str) -> str:
"""Remove common leading indentation from docstring."""
- pass
+ def dummy() -> None:
+ # dummy function to mock `inspect.getdoc`.
+ pass
+
+ dummy.__doc__ = s
+ docstring = inspect.getdoc(dummy)
+ if docstring:
+ return docstring.lstrip("\r\n").rstrip("\r\n")
+ else:
+ return ""
class Token:
"""Better token wrapper for tokenize module."""
- def __init__(self, kind: int, value: Any, start: tuple[int, int], end:
- tuple[int, int], source: str) ->None:
+ def __init__(self, kind: int, value: Any, start: tuple[int, int], end: tuple[int, int],
+ source: str) -> None:
self.kind = kind
self.value = value
self.start = start
self.end = end
self.source = source
- def __eq__(self, other: Any) ->bool:
+ def __eq__(self, other: Any) -> bool:
if isinstance(other, int):
return self.kind == other
elif isinstance(other, str):
@@ -63,38 +115,56 @@ class Token:
else:
raise ValueError('Unknown value: %r' % other)
- def __repr__(self) ->str:
- return (
- f'<Token kind={tokenize.tok_name[self.kind]!r} value={self.value.strip()!r}>'
- )
+ def match(self, *conditions: Any) -> bool:
+ return any(self == candidate for candidate in conditions)
+ def __repr__(self) -> str:
+ return f'<Token kind={tokenize.tok_name[self.kind]!r} value={self.value.strip()!r}>'
-class TokenProcessor:
- def __init__(self, buffers: list[str]) ->None:
+class TokenProcessor:
+ def __init__(self, buffers: list[str]) -> None:
lines = iter(buffers)
self.buffers = buffers
- self.tokens = tokenize.generate_tokens(lambda : next(lines))
+ self.tokens = tokenize.generate_tokens(lambda: next(lines))
self.current: Token | None = None
self.previous: Token | None = None
- def get_line(self, lineno: int) ->str:
+ def get_line(self, lineno: int) -> str:
"""Returns specified line."""
- pass
+ return self.buffers[lineno - 1]
- def fetch_token(self) ->(Token | None):
+ def fetch_token(self) -> Token | None:
"""Fetch the next token from source code.
Returns ``None`` if sequence finished.
"""
- pass
+ try:
+ self.previous = self.current
+ self.current = Token(*next(self.tokens))
+ except StopIteration:
+ self.current = None
+
+ return self.current
- def fetch_until(self, condition: Any) ->list[Token]:
+ def fetch_until(self, condition: Any) -> list[Token]:
"""Fetch tokens until specified token appeared.
.. note:: This also handles parenthesis well.
"""
- pass
+ tokens = []
+ while current := self.fetch_token():
+ tokens.append(current)
+ if current == condition:
+ break
+ if current == [OP, '(']:
+ tokens += self.fetch_until([OP, ')'])
+ elif current == [OP, '{']:
+ tokens += self.fetch_until([OP, '}'])
+ elif current == [OP, '[']:
+ tokens += self.fetch_until([OP, ']'])
+
+ return tokens
class AfterCommentParser(TokenProcessor):
@@ -104,23 +174,51 @@ class AfterCommentParser(TokenProcessor):
and returns the comment for the variable if one exists.
"""
- def __init__(self, lines: list[str]) ->None:
+ def __init__(self, lines: list[str]) -> None:
super().__init__(lines)
self.comment: str | None = None
- def fetch_rvalue(self) ->list[Token]:
+ def fetch_rvalue(self) -> list[Token]:
"""Fetch right-hand value of assignment."""
- pass
-
- def parse(self) ->None:
+ tokens = []
+ while current := self.fetch_token():
+ tokens.append(current)
+ if current == [OP, '(']:
+ tokens += self.fetch_until([OP, ')'])
+ elif current == [OP, '{']:
+ tokens += self.fetch_until([OP, '}'])
+ elif current == [OP, '[']:
+ tokens += self.fetch_until([OP, ']'])
+ elif current == INDENT:
+ tokens += self.fetch_until(DEDENT)
+ elif current == [OP, ';']: # NoQA: SIM114
+ break
+ elif current and current.kind not in {OP, NAME, NUMBER, STRING}:
+ break
+
+ return tokens
+
+ def parse(self) -> None:
"""Parse the code and obtain comment after assignment."""
- pass
+ # skip lvalue (or whole of AnnAssign)
+ while (tok := self.fetch_token()) and not tok.match([OP, '='], NEWLINE, COMMENT):
+ assert tok
+ assert tok is not None
+
+ # skip rvalue (if exists)
+ if tok == [OP, '=']:
+ self.fetch_rvalue()
+ tok = self.current
+ assert tok is not None
+
+ if tok == COMMENT:
+ self.comment = tok.value
class VariableCommentPicker(ast.NodeVisitor):
"""Python source code parser to pick up variable comments."""
- def __init__(self, buffers: list[str], encoding: str) ->None:
+ def __init__(self, buffers: list[str], encoding: str) -> None:
self.counter = itertools.count()
self.buffers = buffers
self.encoding = encoding
@@ -138,60 +236,231 @@ class VariableCommentPicker(ast.NodeVisitor):
self.typing_overload: str | None = None
super().__init__()
- def get_qualname_for(self, name: str) ->(list[str] | None):
+ def get_qualname_for(self, name: str) -> list[str] | None:
"""Get qualified name for given object as a list of string(s)."""
- pass
-
- def get_self(self) ->(ast.arg | None):
+ if self.current_function:
+ if self.current_classes and self.context[-1] == "__init__":
+ # store variable comments inside __init__ method of classes
+ return self.context[:-1] + [name]
+ else:
+ return None
+ else:
+ return [*self.context, name]
+
+ def add_entry(self, name: str) -> None:
+ qualname = self.get_qualname_for(name)
+ if qualname:
+ self.deforders[".".join(qualname)] = next(self.counter)
+
+ def add_final_entry(self, name: str) -> None:
+ qualname = self.get_qualname_for(name)
+ if qualname:
+ self.finals.append(".".join(qualname))
+
+ def add_overload_entry(self, func: ast.FunctionDef) -> None:
+ # avoid circular import problem
+ from sphinx.util.inspect import signature_from_ast
+ qualname = self.get_qualname_for(func.name)
+ if qualname:
+ overloads = self.overloads.setdefault(".".join(qualname), [])
+ overloads.append(signature_from_ast(func))
+
+ def add_variable_comment(self, name: str, comment: str) -> None:
+ qualname = self.get_qualname_for(name)
+ if qualname:
+ basename = ".".join(qualname[:-1])
+ self.comments[(basename, name)] = comment
+
+ def add_variable_annotation(self, name: str, annotation: ast.AST) -> None:
+ qualname = self.get_qualname_for(name)
+ if qualname:
+ basename = ".".join(qualname[:-1])
+ self.annotations[(basename, name)] = ast_unparse(annotation)
+
+ def is_final(self, decorators: list[ast.expr]) -> bool:
+ final = []
+ if self.typing:
+ final.append('%s.final' % self.typing)
+ if self.typing_final:
+ final.append(self.typing_final)
+
+ for decorator in decorators:
+ try:
+ if ast_unparse(decorator) in final:
+ return True
+ except NotImplementedError:
+ pass
+
+ return False
+
+ def is_overload(self, decorators: list[ast.expr]) -> bool:
+ overload = []
+ if self.typing:
+ overload.append('%s.overload' % self.typing)
+ if self.typing_overload:
+ overload.append(self.typing_overload)
+
+ for decorator in decorators:
+ try:
+ if ast_unparse(decorator) in overload:
+ return True
+ except NotImplementedError:
+ pass
+
+ return False
+
+ def get_self(self) -> ast.arg | None:
"""Returns the name of the first argument if in a function."""
- pass
+ if self.current_function and self.current_function.args.args:
+ return self.current_function.args.args[0]
+ if self.current_function and self.current_function.args.posonlyargs:
+ return self.current_function.args.posonlyargs[0]
+ return None
- def get_line(self, lineno: int) ->str:
+ def get_line(self, lineno: int) -> str:
"""Returns specified line."""
- pass
+ return self.buffers[lineno - 1]
- def visit(self, node: ast.AST) ->None:
+ def visit(self, node: ast.AST) -> None:
"""Updates self.previous to the given node."""
- pass
+ super().visit(node)
+ self.previous = node
- def visit_Import(self, node: ast.Import) ->None:
+ def visit_Import(self, node: ast.Import) -> None:
"""Handles Import node and record the order of definitions."""
- pass
+ for name in node.names:
+ self.add_entry(name.asname or name.name)
- def visit_ImportFrom(self, node: ast.ImportFrom) ->None:
+ if name.name == 'typing':
+ self.typing = name.asname or name.name
+ elif name.name == 'typing.final':
+ self.typing_final = name.asname or name.name
+ elif name.name == 'typing.overload':
+ self.typing_overload = name.asname or name.name
+
+ def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
"""Handles Import node and record the order of definitions."""
- pass
+ for name in node.names:
+ self.add_entry(name.asname or name.name)
- def visit_Assign(self, node: ast.Assign) ->None:
- """Handles Assign node and pick up a variable comment."""
- pass
+ if node.module == 'typing' and name.name == 'final':
+ self.typing_final = name.asname or name.name
+ elif node.module == 'typing' and name.name == 'overload':
+ self.typing_overload = name.asname or name.name
- def visit_AnnAssign(self, node: ast.AnnAssign) ->None:
+ def visit_Assign(self, node: ast.Assign) -> None:
+ """Handles Assign node and pick up a variable comment."""
+ try:
+ targets = get_assign_targets(node)
+ varnames: list[str] = functools.reduce(
+ operator.iadd, [get_lvar_names(t, self=self.get_self()) for t in targets], [])
+ current_line = self.get_line(node.lineno)
+ except TypeError:
+ return # this assignment is not new definition!
+
+ # record annotation
+ if hasattr(node, 'annotation') and node.annotation:
+ for varname in varnames:
+ self.add_variable_annotation(varname, node.annotation)
+ elif hasattr(node, 'type_comment') and node.type_comment:
+ for varname in varnames:
+ self.add_variable_annotation(
+ varname, node.type_comment) # type: ignore[arg-type]
+
+ # check comments after assignment
+ parser = AfterCommentParser([current_line[node.col_offset:]] +
+ self.buffers[node.lineno:])
+ parser.parse()
+ if parser.comment and comment_re.match(parser.comment):
+ for varname in varnames:
+ self.add_variable_comment(varname, comment_re.sub('\\1', parser.comment))
+ self.add_entry(varname)
+ return
+
+ # check comments before assignment
+ if indent_re.match(current_line[:node.col_offset]):
+ comment_lines = []
+ for i in range(node.lineno - 1):
+ before_line = self.get_line(node.lineno - 1 - i)
+ if comment_re.match(before_line):
+ comment_lines.append(comment_re.sub('\\1', before_line))
+ else:
+ break
+
+ if comment_lines:
+ comment = dedent_docstring('\n'.join(reversed(comment_lines)))
+ for varname in varnames:
+ self.add_variable_comment(varname, comment)
+ self.add_entry(varname)
+ return
+
+ # not commented (record deforders only)
+ for varname in varnames:
+ self.add_entry(varname)
+
+ def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
"""Handles AnnAssign node and pick up a variable comment."""
- pass
+ self.visit_Assign(node) # type: ignore[arg-type]
- def visit_Expr(self, node: ast.Expr) ->None:
+ def visit_Expr(self, node: ast.Expr) -> None:
"""Handles Expr node and pick up a comment if string."""
- pass
-
- def visit_Try(self, node: ast.Try) ->None:
+ if (isinstance(self.previous, ast.Assign | ast.AnnAssign) and
+ isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)):
+ try:
+ targets = get_assign_targets(self.previous)
+ varnames = get_lvar_names(targets[0], self.get_self())
+ for varname in varnames:
+ if isinstance(node.value.value, str):
+ docstring = node.value.value
+ else:
+ docstring = node.value.value.decode(self.encoding or 'utf-8')
+
+ self.add_variable_comment(varname, dedent_docstring(docstring))
+ self.add_entry(varname)
+ except TypeError:
+ pass # this assignment is not new definition!
+
+ def visit_Try(self, node: ast.Try) -> None:
"""Handles Try node and processes body and else-clause.
.. note:: pycode parser ignores objects definition in except-clause.
"""
- pass
+ for subnode in node.body:
+ self.visit(subnode)
+ for subnode in node.orelse:
+ self.visit(subnode)
- def visit_ClassDef(self, node: ast.ClassDef) ->None:
+ def visit_ClassDef(self, node: ast.ClassDef) -> None:
"""Handles ClassDef node and set context."""
- pass
-
- def visit_FunctionDef(self, node: ast.FunctionDef) ->None:
+ self.current_classes.append(node.name)
+ self.add_entry(node.name)
+ if self.is_final(node.decorator_list):
+ self.add_final_entry(node.name)
+ self.context.append(node.name)
+ self.previous = node
+ for child in node.body:
+ self.visit(child)
+ self.context.pop()
+ self.current_classes.pop()
+
+ def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
"""Handles FunctionDef node and set context."""
- pass
-
- def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) ->None:
+ if self.current_function is None:
+ self.add_entry(node.name) # should be called before setting self.current_function
+ if self.is_final(node.decorator_list):
+ self.add_final_entry(node.name)
+ if self.is_overload(node.decorator_list):
+ self.add_overload_entry(node)
+ self.context.append(node.name)
+ self.current_function = node
+ for child in node.body:
+ self.visit(child)
+ self.context.pop()
+ self.current_function = None
+
+ def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
"""Handles AsyncFunctionDef node and set context."""
- pass
+ self.visit_FunctionDef(node) # type: ignore[arg-type]
class DefinitionFinder(TokenProcessor):
@@ -199,28 +468,75 @@ class DefinitionFinder(TokenProcessor):
classes and methods.
"""
- def __init__(self, lines: list[str]) ->None:
+ def __init__(self, lines: list[str]) -> None:
super().__init__(lines)
self.decorator: Token | None = None
self.context: list[str] = []
self.indents: list[tuple[str, str | None, int | None]] = []
self.definitions: dict[str, tuple[str, int, int]] = {}
- def add_definition(self, name: str, entry: tuple[str, int, int]) ->None:
+ def add_definition(self, name: str, entry: tuple[str, int, int]) -> None:
"""Add a location of definition."""
- pass
+ if self.indents and self.indents[-1][0] == entry[0] == 'def':
+ # ignore definition of inner function
+ pass
+ else:
+ self.definitions[name] = entry
- def parse(self) ->None:
+ def parse(self) -> None:
"""Parse the code to obtain location of definitions."""
- pass
-
- def parse_definition(self, typ: str) ->None:
+ while True:
+ token = self.fetch_token()
+ if token is None:
+ break
+ if token == COMMENT:
+ pass
+ elif token == [OP, '@'] and (self.previous is None or
+ self.previous.match(NEWLINE, NL, INDENT, DEDENT)):
+ if self.decorator is None:
+ self.decorator = token
+ elif token.match([NAME, 'class']):
+ self.parse_definition('class')
+ elif token.match([NAME, 'def']):
+ self.parse_definition('def')
+ elif token == INDENT:
+ self.indents.append(('other', None, None))
+ elif token == DEDENT:
+ self.finalize_block()
+
+ def parse_definition(self, typ: str) -> None:
"""Parse AST of definition."""
- pass
+ name = self.fetch_token()
+ self.context.append(name.value) # type: ignore[union-attr]
+ funcname = '.'.join(self.context)
+
+ if self.decorator:
+ start_pos = self.decorator.start[0]
+ self.decorator = None
+ else:
+ start_pos = name.start[0] # type: ignore[union-attr]
+
+ self.fetch_until([OP, ':'])
+ if self.fetch_token().match(COMMENT, NEWLINE): # type: ignore[union-attr]
+ self.fetch_until(INDENT)
+ self.indents.append((typ, funcname, start_pos))
+ else:
+ # one-liner
+ self.add_definition(funcname,
+ (typ, start_pos, name.end[0])) # type: ignore[union-attr]
+ self.context.pop()
- def finalize_block(self) ->None:
+ def finalize_block(self) -> None:
"""Finalize definition block."""
- pass
+ definition = self.indents.pop()
+ if definition[0] != 'other':
+ typ, funcname, start_pos = definition
+ end_pos = self.current.end[0] - 1 # type: ignore[union-attr]
+ while emptyline_re.match(self.get_line(end_pos)):
+ end_pos -= 1
+
+ self.add_definition(funcname, (typ, start_pos, end_pos)) # type: ignore[arg-type]
+ self.context.pop()
class Parser:
@@ -229,7 +545,7 @@ class Parser:
This is a better wrapper for ``VariableCommentPicker``.
"""
- def __init__(self, code: str, encoding: str='utf-8') ->None:
+ def __init__(self, code: str, encoding: str = 'utf-8') -> None:
self.code = filter_whitespace(code)
self.encoding = encoding
self.annotations: dict[tuple[str, str], str] = {}
@@ -239,14 +555,24 @@ class Parser:
self.finals: list[str] = []
self.overloads: dict[str, list[Signature]] = {}
- def parse(self) ->None:
+ def parse(self) -> None:
"""Parse the source code."""
- pass
+ self.parse_comments()
+ self.parse_definition()
- def parse_comments(self) ->None:
+ def parse_comments(self) -> None:
"""Parse the code and pick up comments."""
- pass
-
- def parse_definition(self) ->None:
+ tree = ast.parse(self.code, type_comments=True)
+ picker = VariableCommentPicker(self.code.splitlines(True), self.encoding)
+ picker.visit(tree)
+ self.annotations = picker.annotations
+ self.comments = picker.comments
+ self.deforders = picker.deforders
+ self.finals = picker.finals
+ self.overloads = picker.overloads
+
+ def parse_definition(self) -> None:
"""Parse the location of definitions from the code."""
- pass
+ parser = DefinitionFinder(self.code.splitlines(True))
+ parser.parse()
+ self.definitions = parser.definitions
diff --git a/sphinx/pygments_styles.py b/sphinx/pygments_styles.py
index 48959e758..d2266421c 100644
--- a/sphinx/pygments_styles.py
+++ b/sphinx/pygments_styles.py
@@ -1,7 +1,18 @@
"""Sphinx theme specific highlighting styles."""
+
from pygments.style import Style
from pygments.styles.friendly import FriendlyStyle
-from pygments.token import Comment, Error, Generic, Keyword, Name, Number, Operator, String, Whitespace
+from pygments.token import (
+ Comment,
+ Error,
+ Generic,
+ Keyword,
+ Name,
+ Number,
+ Operator,
+ String,
+ Whitespace,
+)
class NoneStyle(Style):
@@ -13,33 +24,73 @@ class SphinxStyle(Style):
Like friendly, but a bit darker to enhance contrast on the green
background.
"""
+
background_color = '#eeffcc'
default_style = ''
- styles = {**FriendlyStyle.styles, Generic.Output: '#333', Comment:
- 'italic #408090', Number: '#208050'}
+
+ styles = {
+ **FriendlyStyle.styles,
+ Generic.Output: '#333',
+ Comment: 'italic #408090',
+ Number: '#208050',
+ }
class PyramidStyle(Style):
"""
Pylons/pyramid pygments style based on friendly style, by Blaise Laflamme.
"""
- background_color = '#f8f8f8'
- default_style = ''
- styles = {Whitespace: '#bbbbbb', Comment: 'italic #60a0b0', Comment.
- Preproc: 'noitalic #007020', Comment.Special: 'noitalic bg:#fff0f0',
- Keyword: 'bold #007020', Keyword.Pseudo: 'nobold', Keyword.Type:
- 'nobold #902000', Operator: '#666666', Operator.Word:
- 'bold #007020', Name.Builtin: '#007020', Name.Function: '#06287e',
- Name.Class: 'bold #0e84b5', Name.Namespace: 'bold #0e84b5', Name.
- Exception: '#007020', Name.Variable: '#bb60d5', Name.Constant:
- '#60add5', Name.Label: 'bold #002070', Name.Entity: 'bold #d55537',
- Name.Attribute: '#0e84b5', Name.Tag: 'bold #062873', Name.Decorator:
- 'bold #555555', String: '#4070a0', String.Doc: 'italic', String.
- Interpol: 'italic #70a0d0', String.Escape: 'bold #4070a0', String.
- Regex: '#235388', String.Symbol: '#517918', String.Other: '#c65d09',
- Number: '#40a070', Generic.Heading: 'bold #000080', Generic.
- Subheading: 'bold #800080', Generic.Deleted: '#A00000', Generic.
- Inserted: '#00A000', Generic.Error: '#FF0000', Generic.Emph:
- 'italic', Generic.Strong: 'bold', Generic.Prompt: 'bold #c65d09',
- Generic.Output: '#888', Generic.Traceback: '#04D', Error:
- '#a40000 bg:#fbe3e4'}
+
+ # work in progress...
+
+ background_color = "#f8f8f8"
+ default_style = ""
+
+ styles = {
+ Whitespace: "#bbbbbb",
+ Comment: "italic #60a0b0",
+ Comment.Preproc: "noitalic #007020",
+ Comment.Special: "noitalic bg:#fff0f0",
+
+ Keyword: "bold #007020",
+ Keyword.Pseudo: "nobold",
+ Keyword.Type: "nobold #902000",
+
+ Operator: "#666666",
+ Operator.Word: "bold #007020",
+
+ Name.Builtin: "#007020",
+ Name.Function: "#06287e",
+ Name.Class: "bold #0e84b5",
+ Name.Namespace: "bold #0e84b5",
+ Name.Exception: "#007020",
+ Name.Variable: "#bb60d5",
+ Name.Constant: "#60add5",
+ Name.Label: "bold #002070",
+ Name.Entity: "bold #d55537",
+ Name.Attribute: "#0e84b5",
+ Name.Tag: "bold #062873",
+ Name.Decorator: "bold #555555",
+
+ String: "#4070a0",
+ String.Doc: "italic",
+ String.Interpol: "italic #70a0d0",
+ String.Escape: "bold #4070a0",
+ String.Regex: "#235388",
+ String.Symbol: "#517918",
+ String.Other: "#c65d09",
+ Number: "#40a070",
+
+ Generic.Heading: "bold #000080",
+ Generic.Subheading: "bold #800080",
+ Generic.Deleted: "#A00000",
+ Generic.Inserted: "#00A000",
+ Generic.Error: "#FF0000",
+ Generic.Emph: "italic",
+ Generic.Strong: "bold",
+ Generic.Prompt: "bold #c65d09",
+ Generic.Output: "#888",
+ Generic.Traceback: "#04D",
+
+ Error: "#a40000 bg:#fbe3e4",
+ }
diff --git a/sphinx/registry.py b/sphinx/registry.py
index dac389ea8..caa8c2daf 100644
--- a/sphinx/registry.py
+++ b/sphinx/registry.py
@@ -1,10 +1,13 @@
"""Sphinx component registry."""
+
from __future__ import annotations
+
import traceback
from importlib import import_module
from importlib.metadata import entry_points
from types import MethodType
from typing import TYPE_CHECKING, Any
+
from sphinx.domains import Domain, Index, ObjType
from sphinx.domains.std import GenericObject, Target
from sphinx.errors import ExtensionError, SphinxError, VersionRequirementError
@@ -15,63 +18,506 @@ from sphinx.parsers import Parser as SphinxParser
from sphinx.roles import XRefRole
from sphinx.util import logging
from sphinx.util.logging import prefixed_warnings
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator, Sequence
+
from docutils import nodes
from docutils.core import Publisher
from docutils.nodes import Element, Node, TextElement
from docutils.parsers import Parser
from docutils.parsers.rst import Directive
from docutils.transforms import Transform
+
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.config import Config
from sphinx.environment import BuildEnvironment
from sphinx.ext.autodoc import Documenter
- from sphinx.util.typing import ExtensionMetadata, RoleFunction, TitleGetter, _ExtensionSetupFunc
+ from sphinx.util.typing import (
+ ExtensionMetadata,
+ RoleFunction,
+ TitleGetter,
+ _ExtensionSetupFunc,
+ )
+
logger = logging.getLogger(__name__)
-EXTENSION_BLACKLIST = {'sphinxjp.themecore': '1.2',
- 'sphinxcontrib-napoleon': '1.3', 'sphinxprettysearchresults': '2.0.0'}
+
+# list of deprecated extensions. Keys are extension name.
+# Values are Sphinx version that merge the extension.
+EXTENSION_BLACKLIST = {
+ "sphinxjp.themecore": "1.2",
+ 'sphinxcontrib-napoleon': '1.3',
+ "sphinxprettysearchresults": "2.0.0",
+}
class SphinxComponentRegistry:
+ def __init__(self) -> None:
+ #: special attrgetter for autodoc; class object -> attrgetter
+ self.autodoc_attrgettrs: dict[type, Callable[[Any, str, Any], Any]] = {}
- def __init__(self) ->None:
- self.autodoc_attrgettrs: dict[type, Callable[[Any, str, Any], Any]] = {
- }
+ #: builders; a dict of builder name -> builder class
self.builders: dict[str, type[Builder]] = {}
+
+ #: autodoc documenters; a dict of documenter name -> documenter class
self.documenters: dict[str, type[Documenter]] = {}
+
+ #: css_files; a list of tuple of filename and attributes
self.css_files: list[tuple[str, dict[str, Any]]] = []
+
+ #: domains; a dict of domain name -> domain class
self.domains: dict[str, type[Domain]] = {}
+
+ #: additional directives for domains
+ #: a dict of domain name -> dict of directive name -> directive
self.domain_directives: dict[str, dict[str, type[Directive]]] = {}
+
+ #: additional indices for domains
+ #: a dict of domain name -> list of index class
self.domain_indices: dict[str, list[type[Index]]] = {}
+
+ #: additional object types for domains
+ #: a dict of domain name -> dict of objtype name -> objtype
self.domain_object_types: dict[str, dict[str, ObjType]] = {}
+
+ #: additional roles for domains
+ #: a dict of domain name -> dict of role name -> role impl.
self.domain_roles: dict[str, dict[str, RoleFunction | XRefRole]] = {}
- self.enumerable_nodes: dict[type[Node], tuple[str, TitleGetter | None]
- ] = {}
- self.html_inline_math_renderers: dict[str, tuple[Callable, Callable |
- None]] = {}
- self.html_block_math_renderers: dict[str, tuple[Callable, Callable |
- None]] = {}
+
+ #: additional enumerable nodes
+ #: a dict of node class -> tuple of figtype and title_getter function
+ self.enumerable_nodes: dict[type[Node], tuple[str, TitleGetter | None]] = {}
+
+ #: HTML inline and block math renderers
+ #: a dict of name -> tuple of visit function and depart function
+ self.html_inline_math_renderers: dict[str,
+ tuple[Callable, Callable | None]] = {}
+ self.html_block_math_renderers: dict[str,
+ tuple[Callable, Callable | None]] = {}
+
+ #: HTML assets
self.html_assets_policy: str = 'per_page'
+
+ #: HTML themes
self.html_themes: dict[str, str] = {}
+
+ #: js_files; list of JS paths or URLs
self.js_files: list[tuple[str | None, dict[str, Any]]] = []
+
+ #: LaTeX packages; list of package names and its options
self.latex_packages: list[tuple[str, str | None]] = []
+
self.latex_packages_after_hyperref: list[tuple[str, str | None]] = []
+
+ #: post transforms; list of transforms
self.post_transforms: list[type[Transform]] = []
+
+ #: source paresrs; file type -> parser class
self.source_parsers: dict[str, type[Parser]] = {}
+
+ #: source suffix: suffix -> file type
self.source_suffix: dict[str, str] = {}
+
+ #: custom translators; builder name -> translator class
self.translators: dict[str, type[nodes.NodeVisitor]] = {}
- self.translation_handlers: dict[str, dict[str, tuple[Callable,
- Callable | None]]] = {}
+
+ #: custom handlers for translators
+ #: a dict of builder name -> dict of node name -> visitor and departure functions
+ self.translation_handlers: dict[str, dict[str, tuple[Callable, Callable | None]]] = {}
+
+ #: additional transforms; list of transforms
self.transforms: list[type[Transform]] = []
+
+ # private cache of Docutils Publishers (file type -> publisher object)
self.publishers: dict[str, Publisher] = {}
- def load_extension(self, app: Sphinx, extname: str) ->None:
+ def add_builder(self, builder: type[Builder], override: bool = False) -> None:
+ logger.debug('[app] adding builder: %r', builder)
+ if not hasattr(builder, 'name'):
+ raise ExtensionError(__('Builder class %s has no "name" attribute') % builder)
+ if builder.name in self.builders and not override:
+ raise ExtensionError(__('Builder %r already exists (in module %s)') %
+ (builder.name, self.builders[builder.name].__module__))
+ self.builders[builder.name] = builder
+
+ def preload_builder(self, app: Sphinx, name: str) -> None:
+ if name is None:
+ return
+
+ if name not in self.builders:
+ builder_entry_points = entry_points(group='sphinx.builders')
+ try:
+ entry_point = builder_entry_points[name]
+ except KeyError as exc:
+ raise SphinxError(__('Builder name %s not registered or available'
+ ' through entry point') % name) from exc
+
+ self.load_extension(app, entry_point.module)
+
+ def create_builder(self, app: Sphinx, name: str, env: BuildEnvironment) -> Builder:
+ if name not in self.builders:
+ raise SphinxError(__('Builder name %s not registered') % name)
+
+ return self.builders[name](app, env)
+
+ def add_domain(self, domain: type[Domain], override: bool = False) -> None:
+ logger.debug('[app] adding domain: %r', domain)
+ if domain.name in self.domains and not override:
+ raise ExtensionError(__('domain %s already registered') % domain.name)
+ self.domains[domain.name] = domain
+
+ def has_domain(self, domain: str) -> bool:
+ return domain in self.domains
+
+ def create_domains(self, env: BuildEnvironment) -> Iterator[Domain]:
+ for DomainClass in self.domains.values():
+ domain = DomainClass(env)
+
+ # transplant components added by extensions
+ domain.directives.update(self.domain_directives.get(domain.name, {}))
+ domain.roles.update(self.domain_roles.get(domain.name, {}))
+ domain.indices.extend(self.domain_indices.get(domain.name, []))
+ for name, objtype in self.domain_object_types.get(domain.name, {}).items():
+ domain.add_object_type(name, objtype)
+
+ yield domain
+
+ def add_directive_to_domain(self, domain: str, name: str,
+ cls: type[Directive], override: bool = False) -> None:
+ logger.debug('[app] adding directive to domain: %r', (domain, name, cls))
+ if domain not in self.domains:
+ raise ExtensionError(__('domain %s not yet registered') % domain)
+
+ directives: dict[str, type[Directive]] = self.domain_directives.setdefault(domain, {})
+ if name in directives and not override:
+ raise ExtensionError(__('The %r directive is already registered to domain %s') %
+ (name, domain))
+ directives[name] = cls
+
+ def add_role_to_domain(self, domain: str, name: str,
+ role: RoleFunction | XRefRole, override: bool = False,
+ ) -> None:
+ logger.debug('[app] adding role to domain: %r', (domain, name, role))
+ if domain not in self.domains:
+ raise ExtensionError(__('domain %s not yet registered') % domain)
+ roles = self.domain_roles.setdefault(domain, {})
+ if name in roles and not override:
+ raise ExtensionError(__('The %r role is already registered to domain %s') %
+ (name, domain))
+ roles[name] = role
+
+ def add_index_to_domain(self, domain: str, index: type[Index],
+ override: bool = False) -> None:
+ logger.debug('[app] adding index to domain: %r', (domain, index))
+ if domain not in self.domains:
+ raise ExtensionError(__('domain %s not yet registered') % domain)
+ indices = self.domain_indices.setdefault(domain, [])
+ if index in indices and not override:
+ raise ExtensionError(__('The %r index is already registered to domain %s') %
+ (index.name, domain))
+ indices.append(index)
+
+ def add_object_type(
+ self,
+ directivename: str,
+ rolename: str,
+ indextemplate: str = '',
+ parse_node: Callable | None = None,
+ ref_nodeclass: type[TextElement] | None = None,
+ objname: str = '',
+ doc_field_types: Sequence = (),
+ override: bool = False,
+ ) -> None:
+ logger.debug('[app] adding object type: %r',
+ (directivename, rolename, indextemplate, parse_node,
+ ref_nodeclass, objname, doc_field_types))
+
+ # create a subclass of GenericObject as the new directive
+ directive = type(directivename,
+ (GenericObject, object),
+ {'indextemplate': indextemplate,
+ 'parse_node': parse_node and staticmethod(parse_node),
+ 'doc_field_types': doc_field_types})
+
+ self.add_directive_to_domain('std', directivename, directive)
+ self.add_role_to_domain('std', rolename, XRefRole(innernodeclass=ref_nodeclass))
+
+ object_types = self.domain_object_types.setdefault('std', {})
+ if directivename in object_types and not override:
+ raise ExtensionError(__('The %r object_type is already registered') %
+ directivename)
+ object_types[directivename] = ObjType(objname or directivename, rolename)
+
+ def add_crossref_type(
+ self,
+ directivename: str,
+ rolename: str,
+ indextemplate: str = '',
+ ref_nodeclass: type[TextElement] | None = None,
+ objname: str = '',
+ override: bool = False,
+ ) -> None:
+ logger.debug('[app] adding crossref type: %r',
+ (directivename, rolename, indextemplate, ref_nodeclass, objname))
+
+ # create a subclass of Target as the new directive
+ directive = type(directivename,
+ (Target, object),
+ {'indextemplate': indextemplate})
+
+ self.add_directive_to_domain('std', directivename, directive)
+ self.add_role_to_domain('std', rolename, XRefRole(innernodeclass=ref_nodeclass))
+
+ object_types = self.domain_object_types.setdefault('std', {})
+ if directivename in object_types and not override:
+ raise ExtensionError(__('The %r crossref_type is already registered') %
+ directivename)
+ object_types[directivename] = ObjType(objname or directivename, rolename)
+
+ def add_source_suffix(self, suffix: str, filetype: str, override: bool = False) -> None:
+ logger.debug('[app] adding source_suffix: %r, %r', suffix, filetype)
+ if suffix in self.source_suffix and not override:
+ raise ExtensionError(__('source_suffix %r is already registered') % suffix)
+ self.source_suffix[suffix] = filetype
+
+ def add_source_parser(self, parser: type[Parser], override: bool = False) -> None:
+ logger.debug('[app] adding search source_parser: %r', parser)
+
+ # create a map from filetype to parser
+ for filetype in parser.supported:
+ if filetype in self.source_parsers and not override:
+ raise ExtensionError(__('source_parser for %r is already registered') %
+ filetype)
+ self.source_parsers[filetype] = parser
+
+ def get_source_parser(self, filetype: str) -> type[Parser]:
+ try:
+ return self.source_parsers[filetype]
+ except KeyError as exc:
+ raise SphinxError(__('Source parser for %s not registered') % filetype) from exc
+
+ def get_source_parsers(self) -> dict[str, type[Parser]]:
+ return self.source_parsers
+
+ def create_source_parser(self, app: Sphinx, filename: str) -> Parser:
+ parser_class = self.get_source_parser(filename)
+ parser = parser_class()
+ if isinstance(parser, SphinxParser):
+ parser.set_application(app)
+ return parser
+
+ def add_translator(self, name: str, translator: type[nodes.NodeVisitor],
+ override: bool = False) -> None:
+ logger.debug('[app] Change of translator for the %s builder.', name)
+ if name in self.translators and not override:
+ raise ExtensionError(__('Translator for %r already exists') % name)
+ self.translators[name] = translator
+
+ def add_translation_handlers(
+ self,
+ node: type[Element],
+ **kwargs: tuple[Callable, Callable | None],
+ ) -> None:
+ logger.debug('[app] adding translation_handlers: %r, %r', node, kwargs)
+ for builder_name, handlers in kwargs.items():
+ translation_handlers = self.translation_handlers.setdefault(builder_name, {})
+ try:
+ visit, depart = handlers # unpack once for assertion
+ translation_handlers[node.__name__] = (visit, depart)
+ except ValueError as exc:
+ raise ExtensionError(
+ __('kwargs for add_node() must be a (visit, depart) '
+ 'function tuple: %r=%r') % (builder_name, handlers),
+ ) from exc
+
+ def get_translator_class(self, builder: Builder) -> type[nodes.NodeVisitor]:
+ try:
+ return self.translators[builder.name]
+ except KeyError:
+ try:
+ return builder.default_translator_class
+ except AttributeError as err:
+ msg = f'translator not found for {builder.name}'
+ raise AttributeError(msg) from err
+
+ def create_translator(self, builder: Builder, *args: Any) -> nodes.NodeVisitor:
+ translator_class = self.get_translator_class(builder)
+ translator = translator_class(*args)
+
+ # transplant handlers for custom nodes to translator instance
+ handlers = self.translation_handlers.get(builder.name, None)
+ if handlers is None:
+ # retry with builder.format
+ handlers = self.translation_handlers.get(builder.format, {})
+
+ for name, (visit, depart) in handlers.items():
+ setattr(translator, 'visit_' + name, MethodType(visit, translator))
+ if depart:
+ setattr(translator, 'depart_' + name, MethodType(depart, translator))
+
+ return translator
+
+ def add_transform(self, transform: type[Transform]) -> None:
+ logger.debug('[app] adding transform: %r', transform)
+ self.transforms.append(transform)
+
+ def get_transforms(self) -> list[type[Transform]]:
+ return self.transforms
+
+ def add_post_transform(self, transform: type[Transform]) -> None:
+ logger.debug('[app] adding post transform: %r', transform)
+ self.post_transforms.append(transform)
+
+ def get_post_transforms(self) -> list[type[Transform]]:
+ return self.post_transforms
+
+ def add_documenter(self, objtype: str, documenter: type[Documenter]) -> None:
+ self.documenters[objtype] = documenter
+
+ def add_autodoc_attrgetter(self, typ: type,
+ attrgetter: Callable[[Any, str, Any], Any]) -> None:
+ self.autodoc_attrgettrs[typ] = attrgetter
+
+ def add_css_files(self, filename: str, **attributes: Any) -> None:
+ self.css_files.append((filename, attributes))
+
+ def add_js_file(self, filename: str | None, **attributes: Any) -> None:
+ logger.debug('[app] adding js_file: %r, %r', filename, attributes)
+ self.js_files.append((filename, attributes))
+
+ def has_latex_package(self, name: str) -> bool:
+ packages = self.latex_packages + self.latex_packages_after_hyperref
+ return bool([x for x in packages if x[0] == name])
+
+ def add_latex_package(
+ self, name: str, options: str | None, after_hyperref: bool = False,
+ ) -> None:
+ if self.has_latex_package(name):
+ logger.warning("latex package '%s' already included", name)
+
+ logger.debug('[app] adding latex package: %r', name)
+ if after_hyperref:
+ self.latex_packages_after_hyperref.append((name, options))
+ else:
+ self.latex_packages.append((name, options))
+
+ def add_enumerable_node(
+ self,
+ node: type[Node],
+ figtype: str,
+ title_getter: TitleGetter | None = None, override: bool = False,
+ ) -> None:
+ logger.debug('[app] adding enumerable node: (%r, %r, %r)', node, figtype, title_getter)
+ if node in self.enumerable_nodes and not override:
+ raise ExtensionError(__('enumerable_node %r already registered') % node)
+ self.enumerable_nodes[node] = (figtype, title_getter)
+
+ def add_html_math_renderer(
+ self,
+ name: str,
+ inline_renderers: tuple[Callable, Callable | None] | None,
+ block_renderers: tuple[Callable, Callable | None] | None,
+ ) -> None:
+ logger.debug('[app] adding html_math_renderer: %s, %r, %r',
+ name, inline_renderers, block_renderers)
+ if name in self.html_inline_math_renderers:
+ raise ExtensionError(__('math renderer %s is already registered') % name)
+
+ if inline_renderers is not None:
+ self.html_inline_math_renderers[name] = inline_renderers
+ if block_renderers is not None:
+ self.html_block_math_renderers[name] = block_renderers
+
+ def add_html_theme(self, name: str, theme_path: str) -> None:
+ self.html_themes[name] = theme_path
+
+ def load_extension(self, app: Sphinx, extname: str) -> None:
"""Load a Sphinx extension."""
- pass
+ if extname in app.extensions: # already loaded
+ return
+ if extname in EXTENSION_BLACKLIST:
+ logger.warning(__('the extension %r was already merged with Sphinx since '
+ 'version %s; this extension is ignored.'),
+ extname, EXTENSION_BLACKLIST[extname])
+ return
+
+ # update loading context
+ prefix = __('while setting up extension %s:') % extname
+ with prefixed_warnings(prefix):
+ try:
+ mod = import_module(extname)
+ except ImportError as err:
+ logger.verbose(__('Original exception:\n') + traceback.format_exc())
+ raise ExtensionError(__('Could not import extension %s') % extname,
+ err) from err
+
+ setup: _ExtensionSetupFunc | None = getattr(mod, 'setup', None)
+ if setup is None:
+ logger.warning(__('extension %r has no setup() function; is it really '
+ 'a Sphinx extension module?'), extname)
+ metadata: ExtensionMetadata = {}
+ else:
+ try:
+ metadata = setup(app)
+ except VersionRequirementError as err:
+ # add the extension name to the version required
+ raise VersionRequirementError(
+ __('The %s extension used by this project needs at least '
+ 'Sphinx v%s; it therefore cannot be built with this '
+ 'version.') % (extname, err),
+ ) from err
+
+ if metadata is None:
+ metadata = {}
+ elif not isinstance(metadata, dict):
+ logger.warning(__('extension %r returned an unsupported object from '
+ 'its setup() function; it should return None or a '
+ 'metadata dictionary'), extname)
+ metadata = {}
+
+ app.extensions[extname] = Extension(extname, mod, **metadata)
+
+ def get_envversion(self, app: Sphinx) -> dict[str, int]:
+ from sphinx.environment import ENV_VERSION
+ envversion = {ext.name: ext.metadata['env_version'] for ext in app.extensions.values()
+ if ext.metadata.get('env_version')}
+ envversion['sphinx'] = ENV_VERSION
+ return envversion
+ def get_publisher(self, app: Sphinx, filetype: str) -> Publisher:
+ try:
+ return self.publishers[filetype]
+ except KeyError:
+ pass
+ publisher = create_publisher(app, filetype)
+ self.publishers[filetype] = publisher
+ return publisher
-def merge_source_suffix(app: Sphinx, config: Config) ->None:
+
+def merge_source_suffix(app: Sphinx, config: Config) -> None:
"""Merge any user-specified source_suffix with any added by extensions."""
- pass
+ for suffix, filetype in app.registry.source_suffix.items():
+ if suffix not in app.config.source_suffix: # NoQA: SIM114
+ app.config.source_suffix[suffix] = filetype
+ elif app.config.source_suffix[suffix] == 'restructuredtext':
+ # The filetype is not specified (default filetype).
+ # So it overrides default filetype by extensions setting.
+ app.config.source_suffix[suffix] = filetype
+ elif app.config.source_suffix[suffix] is None:
+ msg = __('`None` is not a valid filetype for %r.') % suffix
+ logger.warning(msg)
+ app.config.source_suffix[suffix] = filetype
+
+ # copy config.source_suffix to registry
+ app.registry.source_suffix = app.config.source_suffix
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.connect('config-inited', merge_source_suffix, priority=800)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/roles.py b/sphinx/roles.py
index 8173b137a..182e2c0da 100644
--- a/sphinx/roles.py
+++ b/sphinx/roles.py
@@ -1,26 +1,44 @@
"""Handlers for additional ReST roles."""
+
from __future__ import annotations
+
import re
from typing import TYPE_CHECKING, Any
+
import docutils.parsers.rst.directives
import docutils.parsers.rst.roles
import docutils.parsers.rst.states
from docutils import nodes, utils
+
from sphinx import addnodes
from sphinx.locale import _, __
from sphinx.util import ws_re
from sphinx.util.docutils import ReferenceRole, SphinxRole
+
if TYPE_CHECKING:
from collections.abc import Sequence
+
from docutils.nodes import Element, Node, TextElement, system_message
+
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import ExtensionMetadata, RoleFunction
-generic_docroles = {'command': addnodes.literal_strong, 'dfn': nodes.
- emphasis, 'kbd': nodes.literal, 'mailheader': addnodes.literal_emphasis,
- 'makevar': addnodes.literal_strong, 'mimetype': addnodes.
- literal_emphasis, 'newsgroup': addnodes.literal_emphasis, 'program':
- addnodes.literal_strong, 'regexp': nodes.literal}
+
+
+generic_docroles = {
+ 'command': addnodes.literal_strong,
+ 'dfn': nodes.emphasis,
+ 'kbd': nodes.literal,
+ 'mailheader': addnodes.literal_emphasis,
+ 'makevar': addnodes.literal_strong,
+ 'mimetype': addnodes.literal_emphasis,
+ 'newsgroup': addnodes.literal_emphasis,
+ 'program': addnodes.literal_strong, # XXX should be an x-ref
+ 'regexp': nodes.literal,
+}
+
+
+# -- generic cross-reference role ----------------------------------------------
class XRefRole(ReferenceRole):
@@ -46,12 +64,18 @@ class XRefRole(ReferenceRole):
* Subclassing and overwriting `process_link()` and/or `result_nodes()`.
"""
+
nodeclass: type[Element] = addnodes.pending_xref
innernodeclass: type[TextElement] = nodes.literal
- def __init__(self, fix_parens: bool=False, lowercase: bool=False,
- nodeclass: (type[Element] | None)=None, innernodeclass: (type[
- TextElement] | None)=None, warn_dangling: bool=False) ->None:
+ def __init__(
+ self,
+ fix_parens: bool = False,
+ lowercase: bool = False,
+ nodeclass: type[Element] | None = None,
+ innernodeclass: type[TextElement] | None = None,
+ warn_dangling: bool = False,
+ ) -> None:
self.fix_parens = fix_parens
self.lowercase = lowercase
self.warn_dangling = warn_dangling
@@ -59,64 +83,405 @@ class XRefRole(ReferenceRole):
self.nodeclass = nodeclass
if innernodeclass is not None:
self.innernodeclass = innernodeclass
+
super().__init__()
- def process_link(self, env: BuildEnvironment, refnode: Element,
- has_explicit_title: bool, title: str, target: str) ->tuple[str, str]:
+ def update_title_and_target(self, title: str, target: str) -> tuple[str, str]:
+ if not self.has_explicit_title:
+ if title.endswith('()'):
+ # remove parentheses
+ title = title[:-2]
+ if self.config.add_function_parentheses:
+ # add them back to all occurrences if configured
+ title += '()'
+ # remove parentheses from the target too
+ if target.endswith('()'):
+ target = target[:-2]
+ return title, target
+
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ if ':' not in self.name:
+ self.refdomain, self.reftype = '', self.name
+ self.classes = ['xref', self.reftype]
+ else:
+ self.refdomain, self.reftype = self.name.split(':', 1)
+ self.classes = ['xref', self.refdomain, f'{self.refdomain}-{self.reftype}']
+
+ if self.disabled:
+ return self.create_non_xref_node()
+ else:
+ return self.create_xref_node()
+
+ def create_non_xref_node(self) -> tuple[list[Node], list[system_message]]:
+ text = utils.unescape(self.text[1:])
+ if self.fix_parens:
+ self.has_explicit_title = False # treat as implicit
+ text, target = self.update_title_and_target(text, '')
+
+ node = self.innernodeclass(self.rawtext, text, classes=self.classes)
+ return self.result_nodes(self.inliner.document, self.env, node, is_ref=False)
+
+ def create_xref_node(self) -> tuple[list[Node], list[system_message]]:
+ target = self.target
+ title = self.title
+ if self.lowercase:
+ target = target.lower()
+ if self.fix_parens:
+ title, target = self.update_title_and_target(title, target)
+
+ # create the reference node
+ options = {
+ 'refdoc': self.env.docname,
+ 'refdomain': self.refdomain,
+ 'reftype': self.reftype,
+ 'refexplicit': self.has_explicit_title,
+ 'refwarn': self.warn_dangling,
+ }
+ refnode = self.nodeclass(self.rawtext, **options)
+ self.set_source_info(refnode)
+
+ # determine the target and title for the class
+ title, target = self.process_link(
+ self.env, refnode, self.has_explicit_title, title, target
+ )
+ refnode['reftarget'] = target
+ refnode += self.innernodeclass(self.rawtext, title, classes=self.classes)
+
+ return self.result_nodes(self.inliner.document, self.env, refnode, is_ref=True)
+
+ # methods that can be overwritten
+
+ def process_link(
+ self,
+ env: BuildEnvironment,
+ refnode: Element,
+ has_explicit_title: bool,
+ title: str,
+ target: str,
+ ) -> tuple[str, str]:
"""Called after parsing title and target text, and creating the
reference node (given in *refnode*). This method can alter the
reference node and must return a new (or the same) ``(title, target)``
tuple.
"""
- pass
+ return title, ws_re.sub(' ', target)
- def result_nodes(self, document: nodes.document, env: BuildEnvironment,
- node: Element, is_ref: bool) ->tuple[list[Node], list[system_message]]:
+ def result_nodes(
+ self,
+ document: nodes.document,
+ env: BuildEnvironment,
+ node: Element,
+ is_ref: bool,
+ ) -> tuple[list[Node], list[system_message]]:
"""Called before returning the finished nodes. *node* is the reference
node if one was created (*is_ref* is then true), else the content node.
This method can add other nodes and must return a ``(nodes, messages)``
tuple (the usual return value of a role function).
"""
- pass
+ return [node], []
class AnyXRefRole(XRefRole):
- pass
+ def process_link(
+ self,
+ env: BuildEnvironment,
+ refnode: Element,
+ has_explicit_title: bool,
+ title: str,
+ target: str,
+ ) -> tuple[str, str]:
+ result = super().process_link(env, refnode, has_explicit_title, title, target)
+ # add all possible context info (i.e. std:program, py:module etc.)
+ refnode.attributes.update(env.ref_context)
+ return result
class PEP(ReferenceRole):
- pass
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ target_id = 'index-%s' % self.env.new_serialno('index')
+ entries = [
+ (
+ 'single',
+ _('Python Enhancement Proposals; PEP %s') % self.target,
+ target_id,
+ '',
+ None,
+ )
+ ]
+
+ index = addnodes.index(entries=entries)
+ target = nodes.target('', '', ids=[target_id])
+ self.inliner.document.note_explicit_target(target)
+
+ try:
+ refuri = self.build_uri()
+ reference = nodes.reference(
+ '', '', internal=False, refuri=refuri, classes=['pep']
+ )
+ if self.has_explicit_title:
+ reference += nodes.strong(self.title, self.title)
+ else:
+ title = 'PEP ' + self.title
+ reference += nodes.strong(title, title)
+ except ValueError:
+ msg = self.inliner.reporter.error(
+ __('invalid PEP number %s') % self.target, line=self.lineno
+ )
+ prb = self.inliner.problematic(self.rawtext, self.rawtext, msg)
+ return [prb], [msg]
+
+ return [index, target, reference], []
+
+ def build_uri(self) -> str:
+ base_url = self.inliner.document.settings.pep_base_url
+ ret = self.target.split('#', 1)
+ if len(ret) == 2:
+ return base_url + 'pep-%04d/#%s' % (int(ret[0]), ret[1])
+ else:
+ return base_url + 'pep-%04d/' % int(ret[0])
class RFC(ReferenceRole):
- pass
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ target_id = 'index-%s' % self.env.new_serialno('index')
+ entries = [('single', 'RFC; RFC %s' % self.target, target_id, '', None)]
+
+ index = addnodes.index(entries=entries)
+ target = nodes.target('', '', ids=[target_id])
+ self.inliner.document.note_explicit_target(target)
+
+ try:
+ refuri = self.build_uri()
+ reference = nodes.reference(
+ '', '', internal=False, refuri=refuri, classes=['rfc']
+ )
+ if self.has_explicit_title:
+ reference += nodes.strong(self.title, self.title)
+ else:
+ title = 'RFC ' + self.title
+ reference += nodes.strong(title, title)
+ except ValueError:
+ msg = self.inliner.reporter.error(
+ __('invalid RFC number %s') % self.target, line=self.lineno
+ )
+ prb = self.inliner.problematic(self.rawtext, self.rawtext, msg)
+ return [prb], [msg]
+
+ return [index, target, reference], []
+
+ def build_uri(self) -> str:
+ base_url = self.inliner.document.settings.rfc_base_url
+ ret = self.target.split('#', 1)
+ if len(ret) == 2:
+ return base_url + self.inliner.rfc_url % int(ret[0]) + '#' + ret[1]
+ else:
+ return base_url + self.inliner.rfc_url % int(ret[0])
class GUILabel(SphinxRole):
- amp_re = re.compile('(?<!&)&(?![&\\s])')
+ amp_re = re.compile(r'(?<!&)&(?![&\s])')
+
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ node = nodes.inline(rawtext=self.rawtext, classes=[self.name])
+ spans = self.amp_re.split(self.text)
+ node += nodes.Text(spans.pop(0))
+ for span in spans:
+ span = span.replace('&&', '&')
+
+ letter = nodes.Text(span[0])
+ accelerator = nodes.inline('', '', letter, classes=['accelerator'])
+ node += accelerator
+ node += nodes.Text(span[1:])
+
+ return [node], []
class MenuSelection(GUILabel):
- BULLET_CHARACTER = '‣'
+ BULLET_CHARACTER = '\N{TRIANGULAR BULLET}'
+
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ self.text = self.text.replace('-->', self.BULLET_CHARACTER)
+ return super().run()
class EmphasizedLiteral(SphinxRole):
- parens_re = re.compile('(\\\\\\\\|\\\\{|\\\\}|{|})')
+ parens_re = re.compile(r'(\\\\|\\{|\\}|{|})')
+
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ children = self.parse(self.text)
+ node = nodes.literal(
+ self.rawtext, '', *children, role=self.name.lower(), classes=[self.name]
+ )
+
+ return [node], []
+
+ def parse(self, text: str) -> list[Node]:
+ result: list[Node] = []
+
+ stack = ['']
+ for part in self.parens_re.split(text):
+ if part == '\\\\': # escaped backslash
+ stack[-1] += '\\'
+ elif part == '{':
+ if len(stack) >= 2 and stack[-2] == '{': # nested
+ stack[-1] += '{'
+ else:
+ # start emphasis
+ stack.extend(('{', ''))
+ elif part == '}':
+ if len(stack) == 3 and stack[1] == '{' and len(stack[2]) > 0:
+ # emphasized word found
+ if stack[0]:
+ result.append(nodes.Text(stack[0]))
+ result.append(nodes.emphasis(stack[2], stack[2]))
+ stack = ['']
+ else:
+ # emphasized word not found; the rparen is not a special symbol
+ stack.append('}')
+ stack = [''.join(stack)]
+ elif part == '\\{': # escaped left-brace
+ stack[-1] += '{'
+ elif part == '\\}': # escaped right-brace
+ stack[-1] += '}'
+ else: # others (containing escaped braces)
+ stack[-1] += part
+
+ if ''.join(stack):
+ # remaining is treated as Text
+ text = ''.join(stack)
+ result.append(nodes.Text(text))
+
+ return result
class Abbreviation(SphinxRole):
- abbr_re = re.compile('\\((.*)\\)$', re.DOTALL)
+ abbr_re = re.compile(r'\((.*)\)$', re.DOTALL)
+
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ options = self.options.copy()
+ matched = self.abbr_re.search(self.text)
+ if matched:
+ text = self.text[: matched.start()].strip()
+ options['explanation'] = matched.group(1)
+ else:
+ text = self.text
+
+ return [nodes.abbreviation(self.rawtext, text, **options)], []
class Manpage(ReferenceRole):
- _manpage_re = re.compile(
- '^(?P<path>(?P<page>.+)[(.](?P<section>[1-9]\\w*)?\\)?)$')
+ _manpage_re = re.compile(r'^(?P<path>(?P<page>.+)[(.](?P<section>[1-9]\w*)?\)?)$')
+
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ manpage = ws_re.sub(' ', self.target)
+ if m := self._manpage_re.match(manpage):
+ info = m.groupdict()
+ else:
+ info = {'path': manpage, 'page': manpage, 'section': ''}
+
+ inner: nodes.Node
+ text = self.title[1:] if self.disabled else self.title
+ if not self.disabled and self.config.manpages_url:
+ uri = self.config.manpages_url.format_map(info)
+ inner = nodes.reference('', text, classes=[self.name], refuri=uri)
+ else:
+ inner = nodes.Text(text)
+ node = addnodes.manpage(self.rawtext, '', inner, classes=[self.name], **info)
+
+ return [node], []
+
+
+# Sphinx provides the `code-block` directive for highlighting code blocks.
+# Docutils provides the `code` role which in theory can be used similarly by
+# defining a custom role for a given programming language:
+#
+# .. .. role:: python(code)
+# :language: python
+# :class: highlight
+#
+# In practice this does not produce correct highlighting because it uses a
+# separate highlighting mechanism that results in the "long" pygments class
+# names rather than "short" pygments class names produced by the Sphinx
+# `code-block` directive and for which this extension contains CSS rules.
+#
+# In addition, even if that issue is fixed, because the highlighting
+# implementation in docutils, despite being based on pygments, differs from that
+# used by Sphinx, the output does not exactly match that produced by the Sphinx
+# `code-block` directive.
+#
+# This issue is noted here: //github.com/sphinx-doc/sphinx/issues/5157
+#
+# This overrides the docutils `code` role to perform highlighting in the same
+# way as the Sphinx `code-block` directive.
+#
+# TODO: Change to use `SphinxRole` once SphinxRole is fixed to support options.
+def code_role(
+ name: str,
+ rawtext: str,
+ text: str,
+ lineno: int,
+ inliner: docutils.parsers.rst.states.Inliner,
+ options: dict[str, Any] | None = None,
+ content: Sequence[str] = (),
+) -> tuple[list[Node], list[system_message]]:
+ if options is None:
+ options = {}
+ options = options.copy()
+ docutils.parsers.rst.roles.set_classes(options)
+ language = options.get('language', '')
+ classes = ['code']
+ if language:
+ classes.append('highlight')
+ if 'classes' in options:
+ classes.extend(options['classes'])
+
+ if language and language not in classes:
+ classes.append(language)
+
+ node = nodes.literal(rawtext, text, classes=classes, language=language)
+
+ return [node], []
+
+
+code_role.options = { # type: ignore[attr-defined]
+ 'class': docutils.parsers.rst.directives.class_option,
+ 'language': docutils.parsers.rst.directives.unchanged,
+}
+
+
+specific_docroles: dict[str, RoleFunction] = {
+ # links to download references
+ 'download': XRefRole(nodeclass=addnodes.download_reference),
+ # links to anything
+ 'any': AnyXRefRole(warn_dangling=True),
+ 'pep': PEP(),
+ 'rfc': RFC(),
+ 'guilabel': GUILabel(),
+ 'menuselection': MenuSelection(),
+ 'file': EmphasizedLiteral(),
+ 'samp': EmphasizedLiteral(),
+ 'abbr': Abbreviation(),
+ 'manpage': Manpage(),
+}
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ from docutils.parsers.rst import roles
+
+ for rolename, nodeclass in generic_docroles.items():
+ generic = roles.GenericRole(rolename, nodeclass)
+ role = roles.CustomRole(rolename, generic, {'classes': [rolename]}) # type: ignore[arg-type]
+ roles.register_local_role(rolename, role) # type: ignore[arg-type]
+
+ for rolename, func in specific_docroles.items():
+ roles.register_local_role(rolename, func) # type: ignore[arg-type]
+ # Since docutils registers it as a canonical role, override it as a
+ # canonical role as well.
+ roles.register_canonical_role('code', code_role) # type: ignore[arg-type]
-code_role.options = {'class': docutils.parsers.rst.directives.class_option,
- 'language': docutils.parsers.rst.directives.unchanged}
-specific_docroles: dict[str, RoleFunction] = {'download': XRefRole(
- nodeclass=addnodes.download_reference), 'any': AnyXRefRole(
- warn_dangling=True), 'pep': PEP(), 'rfc': RFC(), 'guilabel': GUILabel(),
- 'menuselection': MenuSelection(), 'file': EmphasizedLiteral(), 'samp':
- EmphasizedLiteral(), 'abbr': Abbreviation(), 'manpage': Manpage()}
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/search/da.py b/sphinx/search/da.py
index 2474be663..47c574485 100644
--- a/sphinx/search/da.py
+++ b/sphinx/search/da.py
@@ -1,10 +1,14 @@
"""Danish search language: includes the JS Danish stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-danish_stopwords = parse_stop_word(
- """
+
+danish_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/danish/stop.txt
og | and
i | in
@@ -100,8 +104,7 @@ været | be
thi | for (conj)
jer | you
sådan | such, like this/like that
-"""
- )
+''')
class SearchDanish(SearchLanguage):
@@ -109,3 +112,9 @@ class SearchDanish(SearchLanguage):
language_name = 'Danish'
js_stemmer_rawcode = 'danish-stemmer.js'
stopwords = danish_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('danish')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/de.py b/sphinx/search/de.py
index 861369d3e..dae52c9f8 100644
--- a/sphinx/search/de.py
+++ b/sphinx/search/de.py
@@ -1,10 +1,14 @@
"""German search language: includes the JS German stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-german_stopwords = parse_stop_word(
- """
+
+german_stopwords = parse_stop_word('''
|source: https://snowball.tartarus.org/algorithms/german/stop.txt
aber | but
@@ -283,8 +287,7 @@ zum | zu + dem
zur | zu + der
zwar | indeed
zwischen | between
-"""
- )
+''')
class SearchGerman(SearchLanguage):
@@ -292,3 +295,9 @@ class SearchGerman(SearchLanguage):
language_name = 'German'
js_stemmer_rawcode = 'german-stemmer.js'
stopwords = german_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('german')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/en.py b/sphinx/search/en.py
index c943768d8..a1f06bd3f 100644
--- a/sphinx/search/en.py
+++ b/sphinx/search/en.py
@@ -1,10 +1,14 @@
"""English search language: includes the JS porter stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage
-english_stopwords = set(
- """
+
+english_stopwords = set("""
a and are as at
be but by
for
@@ -14,8 +18,8 @@ of on or
such
that the their then there these they this to
was will with
-"""
- .split())
+""".split())
+
js_porter_stemmer = """
/**
* Porter Stemmer
@@ -135,7 +139,8 @@ var Stemmer = function() {
}
// Step 2
- re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;
+ re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|\
+ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
@@ -157,7 +162,8 @@ var Stemmer = function() {
}
// Step 4
- re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/;
+ re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|\
+iti|ous|ive|ize)$/;
re2 = /^(.+?)(s|t)(ion)$/;
if (re.test(w)) {
var fp = re.exec(w);
@@ -206,3 +212,9 @@ class SearchEnglish(SearchLanguage):
language_name = 'English'
js_stemmer_code = js_porter_stemmer
stopwords = english_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('porter')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/es.py b/sphinx/search/es.py
index 9205836e5..247095b45 100644
--- a/sphinx/search/es.py
+++ b/sphinx/search/es.py
@@ -1,10 +1,14 @@
"""Spanish search language: includes the JS Spanish stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-spanish_stopwords = parse_stop_word(
- """
+
+spanish_stopwords = parse_stop_word('''
|source: https://snowball.tartarus.org/algorithms/spanish/stop.txt
de | from, of
la | the, her
@@ -343,8 +347,7 @@ tenida
tenidos
tenidas
tened
-"""
- )
+''')
class SearchSpanish(SearchLanguage):
@@ -352,3 +355,9 @@ class SearchSpanish(SearchLanguage):
language_name = 'Spanish'
js_stemmer_rawcode = 'spanish-stemmer.js'
stopwords = spanish_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('spanish')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/fi.py b/sphinx/search/fi.py
index a5f713684..5eca6e384 100644
--- a/sphinx/search/fi.py
+++ b/sphinx/search/fi.py
@@ -1,10 +1,14 @@
"""Finnish search language: includes the JS Finnish stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-finnish_stopwords = parse_stop_word(
- """
+
+finnish_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/finnish/stop.txt
| forms of BE
@@ -93,8 +97,7 @@ kun | when
niin | so
nyt | now
itse | self
-"""
- )
+''')
class SearchFinnish(SearchLanguage):
@@ -102,3 +105,9 @@ class SearchFinnish(SearchLanguage):
language_name = 'Finnish'
js_stemmer_rawcode = 'finnish-stemmer.js'
stopwords = finnish_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('finnish')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/fr.py b/sphinx/search/fr.py
index 8e0f75484..4d41cf442 100644
--- a/sphinx/search/fr.py
+++ b/sphinx/search/fr.py
@@ -1,10 +1,14 @@
"""French search language: includes the JS French stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-french_stopwords = parse_stop_word(
- """
+
+french_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/french/stop.txt
au | a + le
aux | a + les
@@ -179,8 +183,7 @@ quelle | which
quelles | which
sans | without
soi | oneself
-"""
- )
+''')
class SearchFrench(SearchLanguage):
@@ -188,3 +191,9 @@ class SearchFrench(SearchLanguage):
language_name = 'French'
js_stemmer_rawcode = 'french-stemmer.js'
stopwords = french_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('french')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/hu.py b/sphinx/search/hu.py
index 0f9193750..ccd6ebec3 100644
--- a/sphinx/search/hu.py
+++ b/sphinx/search/hu.py
@@ -1,10 +1,14 @@
"""Hungarian search language: includes the JS Hungarian stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-hungarian_stopwords = parse_stop_word(
- """
+
+hungarian_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/hungarian/stop.txt
| prepared by Anna Tordai
a
@@ -206,8 +210,7 @@ vissza
vele
viszont
volna
-"""
- )
+''')
class SearchHungarian(SearchLanguage):
@@ -215,3 +218,9 @@ class SearchHungarian(SearchLanguage):
language_name = 'Hungarian'
js_stemmer_rawcode = 'hungarian-stemmer.js'
stopwords = hungarian_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('hungarian')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/it.py b/sphinx/search/it.py
index a3e7c45ce..8436dfa5b 100644
--- a/sphinx/search/it.py
+++ b/sphinx/search/it.py
@@ -1,10 +1,14 @@
"""Italian search language: includes the JS Italian stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-italian_stopwords = parse_stop_word(
- """
+
+italian_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/italian/stop.txt
ad | a (to) before vowel
al | a + il
@@ -296,8 +300,7 @@ stessi
stesse
stessimo
stessero
-"""
- )
+''')
class SearchItalian(SearchLanguage):
@@ -305,3 +308,9 @@ class SearchItalian(SearchLanguage):
language_name = 'Italian'
js_stemmer_rawcode = 'italian-stemmer.js'
stopwords = italian_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('italian')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/ja.py b/sphinx/search/ja.py
index 703d7260b..7ff663292 100644
--- a/sphinx/search/ja.py
+++ b/sphinx/search/ja.py
@@ -1,41 +1,52 @@
"""Japanese search language: includes routine to split words."""
+
+# Python Version of TinySegmenter
+# (https://chasen.org/~taku/software/TinySegmenter/)
+# TinySegmenter is super compact Japanese tokenizer.
+#
+# TinySegmenter was originally developed by Taku Kudo <taku(at)chasen.org>.
+# Python Version was developed by xnights <programming.magic(at)gmail.com>.
+# For details, see https://programming-magic.com/?id=170
+
from __future__ import annotations
+
import os
import re
import sys
from typing import Any
+
try:
- import MeCab
+ import MeCab # type: ignore[import-not-found]
native_module = True
except ImportError:
native_module = False
+
try:
- import janome.tokenizer
+ import janome.tokenizer # type: ignore[import-not-found]
janome_module = True
except ImportError:
janome_module = False
+
from sphinx.errors import ExtensionError, SphinxError
from sphinx.search import SearchLanguage
from sphinx.util._importer import import_object
class BaseSplitter:
-
- def __init__(self, options: dict[str, str]) ->None:
+ def __init__(self, options: dict[str, str]) -> None:
self.options = options
- def split(self, input: str) ->list[str]:
+ def split(self, input: str) -> list[str]:
"""
:param str input:
:return:
:rtype: list[str]
"""
- pass
+ raise NotImplementedError
class MecabSplitter(BaseSplitter):
-
- def __init__(self, options: dict[str, str]) ->None:
+ def __init__(self, options: dict[str, str]) -> None:
super().__init__(options)
self.ctypes_libmecab: Any = None
self.ctypes_mecab: Any = None
@@ -45,282 +56,444 @@ class MecabSplitter(BaseSplitter):
self.init_native(options)
self.dict_encode = options.get('dic_enc', 'utf-8')
- def __del__(self) ->None:
+ def split(self, input: str) -> list[str]:
+ if native_module:
+ result = self.native.parse(input)
+ else:
+ result = self.ctypes_libmecab.mecab_sparse_tostr(
+ self.ctypes_mecab, input.encode(self.dict_encode))
+ return result.split(' ')
+
+ def init_native(self, options: dict[str, str]) -> None:
+ param = '-Owakati'
+ dict = options.get('dict')
+ if dict:
+ param += ' -d %s' % dict
+ self.native = MeCab.Tagger(param)
+
+ def init_ctypes(self, options: dict[str, str]) -> None:
+ import ctypes.util
+
+ lib = options.get('lib')
+
+ if lib is None:
+ if sys.platform.startswith('win'):
+ libname = 'libmecab.dll'
+ else:
+ libname = 'mecab'
+ libpath = ctypes.util.find_library(libname)
+ elif os.path.basename(lib) == lib:
+ libpath = ctypes.util.find_library(lib)
+ else:
+ libpath = None
+ if os.path.exists(lib):
+ libpath = lib
+ if libpath is None:
+ raise RuntimeError('MeCab dynamic library is not available')
+
+ param = 'mecab -Owakati'
+ dict = options.get('dict')
+ if dict:
+ param += ' -d %s' % dict
+
+ fs_enc = sys.getfilesystemencoding() or sys.getdefaultencoding()
+
+ self.ctypes_libmecab = ctypes.CDLL(libpath)
+ self.ctypes_libmecab.mecab_new2.argtypes = (ctypes.c_char_p,)
+ self.ctypes_libmecab.mecab_new2.restype = ctypes.c_void_p
+ self.ctypes_libmecab.mecab_sparse_tostr.argtypes = (ctypes.c_void_p, ctypes.c_char_p)
+ self.ctypes_libmecab.mecab_sparse_tostr.restype = ctypes.c_char_p
+ self.ctypes_mecab = self.ctypes_libmecab.mecab_new2(param.encode(fs_enc))
+ if self.ctypes_mecab is None:
+ raise SphinxError('mecab initialization failed')
+
+ def __del__(self) -> None:
if self.ctypes_libmecab:
self.ctypes_libmecab.mecab_destroy(self.ctypes_mecab)
class JanomeSplitter(BaseSplitter):
-
- def __init__(self, options: dict[str, str]) ->None:
+ def __init__(self, options: dict[str, str]) -> None:
super().__init__(options)
self.user_dict = options.get('user_dic')
self.user_dict_enc = options.get('user_dic_enc', 'utf8')
self.init_tokenizer()
+ def init_tokenizer(self) -> None:
+ if not janome_module:
+ raise RuntimeError('Janome is not available')
+ self.tokenizer = janome.tokenizer.Tokenizer(udic=self.user_dict, udic_enc=self.user_dict_enc)
+
+ def split(self, input: str) -> list[str]:
+ result = ' '.join(token.surface for token in self.tokenizer.tokenize(input))
+ return result.split(' ')
+
class DefaultSplitter(BaseSplitter):
patterns_ = {re.compile(pattern): value for pattern, value in {
- '[一二三四五六七八九十百千万億兆]': 'M', '[一-龠々〆ヵヶ]': 'H', '[ぁ-ん]': 'I',
- '[ァ-ヴーｱ-ﾝﾞｰ]': 'K', '[a-zA-Zａ-ｚＡ-Ｚ]': 'A', '[0-9０-９]': 'N'}.items()}
+ '[一二三四五六七八九十百千万億兆]': 'M',
+ '[一-龠々〆ヵヶ]': 'H',
+ '[ぁ-ん]': 'I',
+ '[ァ-ヴーｱ-ﾝﾞｰ]': 'K',
+ '[a-zA-Zａ-ｚＡ-Ｚ]': 'A',
+ '[0-9０-９]': 'N',
+ }.items()}
BIAS__ = -332
BC1__ = {'HH': 6, 'II': 2461, 'KH': 406, 'OH': -1378}
BC2__ = {'AA': -3267, 'AI': 2744, 'AN': -878, 'HH': -4070, 'HM': -1711,
- 'HN': 4012, 'HO': 3761, 'IA': 1327, 'IH': -1184, 'II': -1332, 'IK':
- 1721, 'IO': 5492, 'KI': 3831, 'KK': -8741, 'MH': -3132, 'MK': 3334,
- 'OO': -2920}
- BC3__ = {'HH': 996, 'HI': 626, 'HK': -721, 'HN': -1307, 'HO': -836,
- 'IH': -301, 'KK': 2762, 'MK': 1079, 'MM': 4034, 'OA': -1652, 'OH': 266}
+ 'HN': 4012, 'HO': 3761, 'IA': 1327, 'IH': -1184, 'II': -1332,
+ 'IK': 1721, 'IO': 5492, 'KI': 3831, 'KK': -8741, 'MH': -3132,
+ 'MK': 3334, 'OO': -2920}
+ BC3__ = {'HH': 996, 'HI': 626, 'HK': -721, 'HN': -1307, 'HO': -836, 'IH': -301,
+ 'KK': 2762, 'MK': 1079, 'MM': 4034, 'OA': -1652, 'OH': 266}
BP1__ = {'BB': 295, 'OB': 304, 'OO': -125, 'UB': 352}
BP2__ = {'BO': 60, 'OO': -1762}
- BQ1__ = {'BHH': 1150, 'BHM': 1521, 'BII': -1158, 'BIM': 886, 'BMH':
- 1208, 'BNH': 449, 'BOH': -91, 'BOO': -2597, 'OHI': 451, 'OIH': -296,
- 'OKA': 1851, 'OKH': -1020, 'OKK': 904, 'OOO': 2965}
- BQ2__ = {'BHH': 118, 'BHI': -1159, 'BHM': 466, 'BIH': -919, 'BKK': -
- 1720, 'BKO': 864, 'OHH': -1139, 'OHM': -181, 'OIH': 153, 'UHI': -1146}
+ BQ1__ = {'BHH': 1150, 'BHM': 1521, 'BII': -1158, 'BIM': 886, 'BMH': 1208,
+ 'BNH': 449, 'BOH': -91, 'BOO': -2597, 'OHI': 451, 'OIH': -296,
+ 'OKA': 1851, 'OKH': -1020, 'OKK': 904, 'OOO': 2965}
+ BQ2__ = {'BHH': 118, 'BHI': -1159, 'BHM': 466, 'BIH': -919, 'BKK': -1720,
+ 'BKO': 864, 'OHH': -1139, 'OHM': -181, 'OIH': 153, 'UHI': -1146}
BQ3__ = {'BHH': -792, 'BHI': 2664, 'BII': -299, 'BKI': 419, 'BMH': 937,
- 'BMM': 8335, 'BNN': 998, 'BOH': 775, 'OHH': 2174, 'OHM': 439, 'OII':
- 280, 'OKH': 1798, 'OKI': -793, 'OKO': -2242, 'OMH': -2402, 'OOO': 11699
- }
- BQ4__ = {'BHH': -3895, 'BIH': 3761, 'BII': -4654, 'BIK': 1348, 'BKK': -
- 1806, 'BMI': -3385, 'BOO': -12396, 'OAH': 926, 'OHH': 266, 'OHK': -
- 2036, 'ONN': -973}
- BW1__ = {',と': 660, ',同': 727, 'B1あ': 1404, 'B1同': 542, '、と': 660, '、同':
- 727, '」と': 1682, 'あっ': 1505, 'いう': 1743, 'いっ': -2055, 'いる': 672,
- 'うし': -4817, 'うん': 665, 'から': 3472, 'がら': 600, 'こう': -790, 'こと':
- 2083, 'こん': -1262, 'さら': -4143, 'さん': 4573, 'した': 2641, 'して': 1104,
- 'すで': -3399, 'そこ': 1977, 'それ': -871, 'たち': 1122, 'ため': 601, 'った':
- 3463, 'つい': -802, 'てい': 805, 'てき': 1249, 'でき': 1127, 'です': 3445,
- 'では': 844, 'とい': -4915, 'とみ': 1922, 'どこ': 3887, 'ない': 5713, 'なっ':
- 3015, 'など': 7379, 'なん': -1113, 'にし': 2468, 'には': 1498, 'にも': 1671,
- 'に対': -912, 'の一': -501, 'の中': 741, 'ませ': 2448, 'まで': 1711, 'まま':
- 2600, 'まる': -2155, 'やむ': -1947, 'よっ': -2565, 'れた': 2369, 'れで': -913,
- 'をし': 1860, 'を見': 731, '亡く': -1886, '京都': 2558, '取り': -2784, '大き':
- -2604, '大阪': 1497, '平方': -2314, '引き': -1336, '日本': -195, '本当': -
- 2423, '毎日': -2113, '目指': -724, 'Ｂ１あ': 1404, 'Ｂ１同': 542, '｣と': 1682}
- BW2__ = {'..': -11822, '11': -669, '――': -5730, '−−': -13175, 'いう': -
- 1609, 'うか': 2490, 'かし': -1350, 'かも': -602, 'から': -7194, 'かれ': 4612,
- 'がい': 853, 'がら': -3198, 'きた': 1941, 'くな': -1597, 'こと': -8392, 'この':
- -4193, 'させ': 4533, 'され': 13168, 'さん': -3977, 'しい': -1819, 'しか': -
- 545, 'した': 5078, 'して': 972, 'しな': 939, 'その': -3744, 'たい': -1253,
- 'たた': -662, 'ただ': -3857, 'たち': -786, 'たと': 1224, 'たは': -939, 'った':
- 4589, 'って': 1647, 'っと': -2094, 'てい': 6144, 'てき': 3640, 'てく': 2551,
- 'ては': -3110, 'ても': -3065, 'でい': 2666, 'でき': -1528, 'でし': -3828,
- 'です': -4761, 'でも': -4203, 'とい': 1890, 'とこ': -1746, 'とと': -2279,
- 'との': 720, 'とみ': 5168, 'とも': -3941, 'ない': -2488, 'なが': -1313, 'など':
- -6509, 'なの': 2614, 'なん': 3099, 'にお': -1615, 'にし': 2748, 'にな': 2454,
- 'によ': -7236, 'に対': -14943, 'に従': -4688, 'に関': -11388, 'のか': 2093,
- 'ので': -7059, 'のに': -6041, 'のの': -6125, 'はい': 1073, 'はが': -1033,
- 'はず': -2532, 'ばれ': 1813, 'まし': -1316, 'まで': -6621, 'まれ': 5409, 'めて':
- -3153, 'もい': 2230, 'もの': -10713, 'らか': -944, 'らし': -1611, 'らに': -
- 1897, 'りし': 651, 'りま': 1620, 'れた': 4270, 'れて': 849, 'れば': 4114,
- 'ろう': 6067, 'われ': 7901, 'を通': -11877, 'んだ': 728, 'んな': -4115, '一人':
- 602, '一方': -1375, '一日': 970, '一部': -1051, '上が': -4479, '会社': -1116,
- '出て': 2163, '分の': -7758, '同党': 970, '同日': -913, '大阪': -2471, '委員':
- -1250, '少な': -1050, '年度': -8669, '年間': -1626, '府県': -2363, '手権': -
- 1982, '新聞': -4066, '日新': -722, '日本': -7068, '日米': 3372, '曜日': -601,
- '朝鮮': -2355, '本人': -2697, '東京': -1543, '然と': -1384, '社会': -1276,
- '立て': -990, '第に': -1612, '米国': -4268, '１１': -669}
+ 'BMM': 8335, 'BNN': 998, 'BOH': 775, 'OHH': 2174, 'OHM': 439, 'OII': 280,
+ 'OKH': 1798, 'OKI': -793, 'OKO': -2242, 'OMH': -2402, 'OOO': 11699}
+ BQ4__ = {'BHH': -3895, 'BIH': 3761, 'BII': -4654, 'BIK': 1348, 'BKK': -1806,
+ 'BMI': -3385, 'BOO': -12396, 'OAH': 926, 'OHH': 266, 'OHK': -2036,
+ 'ONN': -973}
+ BW1__ = {',と': 660, ',同': 727, 'B1あ': 1404, 'B1同': 542, '、と': 660,
+ '、同': 727, '」と': 1682, 'あっ': 1505, 'いう': 1743, 'いっ': -2055,
+ 'いる': 672, 'うし': -4817, 'うん': 665, 'から': 3472, 'がら': 600,
+ 'こう': -790, 'こと': 2083, 'こん': -1262, 'さら': -4143, 'さん': 4573,
+ 'した': 2641, 'して': 1104, 'すで': -3399, 'そこ': 1977, 'それ': -871,
+ 'たち': 1122, 'ため': 601, 'った': 3463, 'つい': -802, 'てい': 805,
+ 'てき': 1249, 'でき': 1127, 'です': 3445, 'では': 844, 'とい': -4915,
+ 'とみ': 1922, 'どこ': 3887, 'ない': 5713, 'なっ': 3015, 'など': 7379,
+ 'なん': -1113, 'にし': 2468, 'には': 1498, 'にも': 1671, 'に対': -912,
+ 'の一': -501, 'の中': 741, 'ませ': 2448, 'まで': 1711, 'まま': 2600,
+ 'まる': -2155, 'やむ': -1947, 'よっ': -2565, 'れた': 2369, 'れで': -913,
+ 'をし': 1860, 'を見': 731, '亡く': -1886, '京都': 2558, '取り': -2784,
+ '大き': -2604, '大阪': 1497, '平方': -2314, '引き': -1336, '日本': -195,
+ '本当': -2423, '毎日': -2113, '目指': -724, 'Ｂ１あ': 1404, 'Ｂ１同': 542,
+ '｣と': 1682}
+ BW2__ = {'..': -11822, '11': -669, '――': -5730, '−−': -13175, 'いう': -1609,
+ 'うか': 2490, 'かし': -1350, 'かも': -602, 'から': -7194, 'かれ': 4612,
+ 'がい': 853, 'がら': -3198, 'きた': 1941, 'くな': -1597, 'こと': -8392,
+ 'この': -4193, 'させ': 4533, 'され': 13168, 'さん': -3977, 'しい': -1819,
+ 'しか': -545, 'した': 5078, 'して': 972, 'しな': 939, 'その': -3744,
+ 'たい': -1253, 'たた': -662, 'ただ': -3857, 'たち': -786, 'たと': 1224,
+ 'たは': -939, 'った': 4589, 'って': 1647, 'っと': -2094, 'てい': 6144,
+ 'てき': 3640, 'てく': 2551, 'ては': -3110, 'ても': -3065, 'でい': 2666,
+ 'でき': -1528, 'でし': -3828, 'です': -4761, 'でも': -4203, 'とい': 1890,
+ 'とこ': -1746, 'とと': -2279, 'との': 720, 'とみ': 5168, 'とも': -3941,
+ 'ない': -2488, 'なが': -1313, 'など': -6509, 'なの': 2614, 'なん': 3099,
+ 'にお': -1615, 'にし': 2748, 'にな': 2454, 'によ': -7236, 'に対': -14943,
+ 'に従': -4688, 'に関': -11388, 'のか': 2093, 'ので': -7059, 'のに': -6041,
+ 'のの': -6125, 'はい': 1073, 'はが': -1033, 'はず': -2532, 'ばれ': 1813,
+ 'まし': -1316, 'まで': -6621, 'まれ': 5409, 'めて': -3153, 'もい': 2230,
+ 'もの': -10713, 'らか': -944, 'らし': -1611, 'らに': -1897, 'りし': 651,
+ 'りま': 1620, 'れた': 4270, 'れて': 849, 'れば': 4114, 'ろう': 6067,
+ 'われ': 7901, 'を通': -11877, 'んだ': 728, 'んな': -4115, '一人': 602,
+ '一方': -1375, '一日': 970, '一部': -1051, '上が': -4479, '会社': -1116,
+ '出て': 2163, '分の': -7758, '同党': 970, '同日': -913, '大阪': -2471,
+ '委員': -1250, '少な': -1050, '年度': -8669, '年間': -1626, '府県': -2363,
+ '手権': -1982, '新聞': -4066, '日新': -722, '日本': -7068, '日米': 3372,
+ '曜日': -601, '朝鮮': -2355, '本人': -2697, '東京': -1543, '然と': -1384,
+ '社会': -1276, '立て': -990, '第に': -1612, '米国': -4268, '１１': -669}
BW3__ = {'あた': -2194, 'あり': 719, 'ある': 3846, 'い.': -1185, 'い。': -1185,
- 'いい': 5308, 'いえ': 2079, 'いく': 3029, 'いた': 2056, 'いっ': 1883, 'いる':
- 5600, 'いわ': 1527, 'うち': 1117, 'うと': 4798, 'えと': 1454, 'か.': 2857,
- 'か。': 2857, 'かけ': -743, 'かっ': -4098, 'かに': -669, 'から': 6520, 'かり':
- -2670, 'が,': 1816, 'が、': 1816, 'がき': -4855, 'がけ': -1127, 'がっ': -913,
- 'がら': -4977, 'がり': -2064, 'きた': 1645, 'けど': 1374, 'こと': 7397, 'この':
- 1542, 'ころ': -2757, 'さい': -714, 'さを': 976, 'し,': 1557, 'し、': 1557,
- 'しい': -3714, 'した': 3562, 'して': 1449, 'しな': 2608, 'しま': 1200, 'す.':
- -1310, 'す。': -1310, 'する': 6521, 'ず,': 3426, 'ず、': 3426, 'ずに': 841,
- 'そう': 428, 'た.': 8875, 'た。': 8875, 'たい': -594, 'たの': 812, 'たり': -
- 1183, 'たる': -853, 'だ.': 4098, 'だ。': 4098, 'だっ': 1004, 'った': -4748,
- 'って': 300, 'てい': 6240, 'てお': 855, 'ても': 302, 'です': 1437, 'でに': -
- 1482, 'では': 2295, 'とう': -1387, 'とし': 2266, 'との': 541, 'とも': -3543,
- 'どう': 4664, 'ない': 1796, 'なく': -903, 'など': 2135, 'に,': -1021, 'に、':
- -1021, 'にし': 1771, 'にな': 1906, 'には': 2644, 'の,': -724, 'の、': -724,
- 'の子': -1000, 'は,': 1337, 'は、': 1337, 'べき': 2181, 'まし': 1113, 'ます':
- 6943, 'まっ': -1549, 'まで': 6154, 'まれ': -793, 'らし': 1479, 'られ': 6820,
- 'るる': 3818, 'れ,': 854, 'れ、': 854, 'れた': 1850, 'れて': 1375, 'れば': -
- 3246, 'れる': 1091, 'われ': -605, 'んだ': 606, 'んで': 798, 'カ月': 990, '会議':
- 860, '入り': 1232, '大会': 2217, '始め': 1681, '市': 965, '新聞': -5055,
- '日,': 974, '日、': 974, '社会': 2024, 'ｶ月': 990}
+ 'いい': 5308, 'いえ': 2079, 'いく': 3029, 'いた': 2056, 'いっ': 1883,
+ 'いる': 5600, 'いわ': 1527, 'うち': 1117, 'うと': 4798, 'えと': 1454,
+ 'か.': 2857, 'か。': 2857, 'かけ': -743, 'かっ': -4098, 'かに': -669,
+ 'から': 6520, 'かり': -2670, 'が,': 1816, 'が、': 1816, 'がき': -4855,
+ 'がけ': -1127, 'がっ': -913, 'がら': -4977, 'がり': -2064, 'きた': 1645,
+ 'けど': 1374, 'こと': 7397, 'この': 1542, 'ころ': -2757, 'さい': -714,
+ 'さを': 976, 'し,': 1557, 'し、': 1557, 'しい': -3714, 'した': 3562,
+ 'して': 1449, 'しな': 2608, 'しま': 1200, 'す.': -1310, 'す。': -1310,
+ 'する': 6521, 'ず,': 3426, 'ず、': 3426, 'ずに': 841, 'そう': 428,
+ 'た.': 8875, 'た。': 8875, 'たい': -594, 'たの': 812, 'たり': -1183,
+ 'たる': -853, 'だ.': 4098, 'だ。': 4098, 'だっ': 1004, 'った': -4748,
+ 'って': 300, 'てい': 6240, 'てお': 855, 'ても': 302, 'です': 1437,
+ 'でに': -1482, 'では': 2295, 'とう': -1387, 'とし': 2266, 'との': 541,
+ 'とも': -3543, 'どう': 4664, 'ない': 1796, 'なく': -903, 'など': 2135,
+ 'に,': -1021, 'に、': -1021, 'にし': 1771, 'にな': 1906, 'には': 2644,
+ 'の,': -724, 'の、': -724, 'の子': -1000, 'は,': 1337, 'は、': 1337,
+ 'べき': 2181, 'まし': 1113, 'ます': 6943, 'まっ': -1549, 'まで': 6154,
+ 'まれ': -793, 'らし': 1479, 'られ': 6820, 'るる': 3818, 'れ,': 854,
+ 'れ、': 854, 'れた': 1850, 'れて': 1375, 'れば': -3246, 'れる': 1091,
+ 'われ': -605, 'んだ': 606, 'んで': 798, 'カ月': 990, '会議': 860,
+ '入り': 1232, '大会': 2217, '始め': 1681, '市': 965, '新聞': -5055,
+ '日,': 974, '日、': 974, '社会': 2024, 'ｶ月': 990}
TC1__ = {'AAA': 1093, 'HHH': 1029, 'HHM': 580, 'HII': 998, 'HOH': -390,
- 'HOM': -331, 'IHI': 1169, 'IOH': -142, 'IOI': -1015, 'IOM': 467,
- 'MMH': 187, 'OOI': -1832}
- TC2__ = {'HHO': 2088, 'HII': -1023, 'HMM': -1154, 'IHI': -1965, 'KKH':
- 703, 'OII': -2649}
+ 'HOM': -331, 'IHI': 1169, 'IOH': -142, 'IOI': -1015, 'IOM': 467,
+ 'MMH': 187, 'OOI': -1832}
+ TC2__ = {'HHO': 2088, 'HII': -1023, 'HMM': -1154, 'IHI': -1965,
+ 'KKH': 703, 'OII': -2649}
TC3__ = {'AAA': -294, 'HHH': 346, 'HHI': -341, 'HII': -1088, 'HIK': 731,
- 'HOH': -1486, 'IHH': 128, 'IHI': -3041, 'IHO': -1935, 'IIH': -825,
- 'IIM': -1035, 'IOI': -542, 'KHH': -1216, 'KKA': 491, 'KKH': -1217,
- 'KOK': -1009, 'MHH': -2694, 'MHM': -457, 'MHO': 123, 'MMH': -471,
- 'NNH': -1689, 'NNO': 662, 'OHO': -3393}
+ 'HOH': -1486, 'IHH': 128, 'IHI': -3041, 'IHO': -1935, 'IIH': -825,
+ 'IIM': -1035, 'IOI': -542, 'KHH': -1216, 'KKA': 491, 'KKH': -1217,
+ 'KOK': -1009, 'MHH': -2694, 'MHM': -457, 'MHO': 123, 'MMH': -471,
+ 'NNH': -1689, 'NNO': 662, 'OHO': -3393}
TC4__ = {'HHH': -203, 'HHI': 1344, 'HHK': 365, 'HHM': -122, 'HHN': 182,
- 'HHO': 669, 'HIH': 804, 'HII': 679, 'HOH': 446, 'IHH': 695, 'IHO':
- -2324, 'IIH': 321, 'III': 1497, 'IIO': 656, 'IOO': 54, 'KAK': 4845,
- 'KKA': 3386, 'KKK': 3065, 'MHH': -405, 'MHI': 201, 'MMH': -241,
- 'MMM': 661, 'MOM': 841}
- TQ1__ = {'BHHH': -227, 'BHHI': 316, 'BHIH': -132, 'BIHH': 60, 'BIII':
- 1595, 'BNHH': -744, 'BOHH': 225, 'BOOO': -908, 'OAKK': 482, 'OHHH':
- 281, 'OHIH': 249, 'OIHI': 200, 'OIIH': -68}
+ 'HHO': 669, 'HIH': 804, 'HII': 679, 'HOH': 446, 'IHH': 695,
+ 'IHO': -2324, 'IIH': 321, 'III': 1497, 'IIO': 656, 'IOO': 54,
+ 'KAK': 4845, 'KKA': 3386, 'KKK': 3065, 'MHH': -405, 'MHI': 201,
+ 'MMH': -241, 'MMM': 661, 'MOM': 841}
+ TQ1__ = {'BHHH': -227, 'BHHI': 316, 'BHIH': -132, 'BIHH': 60, 'BIII': 1595,
+ 'BNHH': -744, 'BOHH': 225, 'BOOO': -908, 'OAKK': 482, 'OHHH': 281,
+ 'OHIH': 249, 'OIHI': 200, 'OIIH': -68}
TQ2__ = {'BIHH': -1401, 'BIII': -1033, 'BKAK': -543, 'BOOO': -5591}
- TQ3__ = {'BHHH': 478, 'BHHM': -1073, 'BHIH': 222, 'BHII': -504, 'BIIH':
- -116, 'BIII': -105, 'BMHI': -863, 'BMHM': -464, 'BOMH': 620, 'OHHH':
- 346, 'OHHI': 1729, 'OHII': 997, 'OHMH': 481, 'OIHH': 623, 'OIIH':
- 1344, 'OKAK': 2792, 'OKHH': 587, 'OKKA': 679, 'OOHH': 110, 'OOII': -685
- }
- TQ4__ = {'BHHH': -721, 'BHHM': -3604, 'BHII': -966, 'BIIH': -607,
- 'BIII': -2181, 'OAAA': -2763, 'OAKK': 180, 'OHHH': -294, 'OHHI':
- 2446, 'OHHO': 480, 'OHIH': -1573, 'OIHH': 1935, 'OIHI': -493,
- 'OIIH': 626, 'OIII': -4007, 'OKAK': -8156}
+ TQ3__ = {'BHHH': 478, 'BHHM': -1073, 'BHIH': 222, 'BHII': -504, 'BIIH': -116,
+ 'BIII': -105, 'BMHI': -863, 'BMHM': -464, 'BOMH': 620, 'OHHH': 346,
+ 'OHHI': 1729, 'OHII': 997, 'OHMH': 481, 'OIHH': 623, 'OIIH': 1344,
+ 'OKAK': 2792, 'OKHH': 587, 'OKKA': 679, 'OOHH': 110, 'OOII': -685}
+ TQ4__ = {'BHHH': -721, 'BHHM': -3604, 'BHII': -966, 'BIIH': -607, 'BIII': -2181,
+ 'OAAA': -2763, 'OAKK': 180, 'OHHH': -294, 'OHHI': 2446, 'OHHO': 480,
+ 'OHIH': -1573, 'OIHH': 1935, 'OIHI': -493, 'OIIH': 626, 'OIII': -4007,
+ 'OKAK': -8156}
TW1__ = {'につい': -4681, '東京都': 2026}
- TW2__ = {'ある程': -2049, 'いった': -1256, 'ころが': -2434, 'しょう': 3873, 'その後':
- -4430, 'だって': -1049, 'ていた': 1833, 'として': -4657, 'ともに': -4517, 'もので':
- 1882, '一気に': -792, '初めて': -1512, '同時に': -8097, '大きな': -1255, '対して':
- -2721, '社会党': -3216}
- TW3__ = {'いただ': -1734, 'してい': 1314, 'として': -4314, 'につい': -5483, 'にとっ':
- -5989, 'に当た': -6247, 'ので,': -727, 'ので、': -727, 'のもの': -600, 'れから':
- -3752, '十二月': -2287}
- TW4__ = {'いう.': 8576, 'いう。': 8576, 'からな': -2348, 'してい': 2958, 'たが,':
- 1516, 'たが、': 1516, 'ている': 1538, 'という': 1349, 'ました': 5543, 'ません':
- 1097, 'ようと': -4258, 'よると': 5865}
+ TW2__ = {'ある程': -2049, 'いった': -1256, 'ころが': -2434, 'しょう': 3873,
+ 'その後': -4430, 'だって': -1049, 'ていた': 1833, 'として': -4657,
+ 'ともに': -4517, 'もので': 1882, '一気に': -792, '初めて': -1512,
+ '同時に': -8097, '大きな': -1255, '対して': -2721, '社会党': -3216}
+ TW3__ = {'いただ': -1734, 'してい': 1314, 'として': -4314, 'につい': -5483,
+ 'にとっ': -5989, 'に当た': -6247, 'ので,': -727, 'ので、': -727,
+ 'のもの': -600, 'れから': -3752, '十二月': -2287}
+ TW4__ = {'いう.': 8576, 'いう。': 8576, 'からな': -2348, 'してい': 2958,
+ 'たが,': 1516, 'たが、': 1516, 'ている': 1538, 'という': 1349,
+ 'ました': 5543, 'ません': 1097, 'ようと': -4258, 'よると': 5865}
UC1__ = {'A': 484, 'K': 93, 'M': 645, 'O': -505}
UC2__ = {'A': 819, 'H': 1059, 'I': 409, 'M': 3987, 'N': 5775, 'O': 646}
UC3__ = {'A': -1370, 'I': 2311}
- UC4__ = {'A': -2643, 'H': 1809, 'I': -1032, 'K': -3450, 'M': 3565, 'N':
- 3876, 'O': 6646}
+ UC4__ = {'A': -2643, 'H': 1809, 'I': -1032, 'K': -3450, 'M': 3565,
+ 'N': 3876, 'O': 6646}
UC5__ = {'H': 313, 'I': -1238, 'K': -799, 'M': 539, 'O': -831}
UC6__ = {'H': -506, 'I': -253, 'K': 87, 'M': 247, 'O': -387}
UP1__ = {'O': -214}
UP2__ = {'B': 69, 'O': 935}
UP3__ = {'B': 189}
- UQ1__ = {'BH': 21, 'BI': -12, 'BK': -99, 'BN': 142, 'BO': -56, 'OH': -
- 95, 'OI': 477, 'OK': 410, 'OO': -2422}
+ UQ1__ = {'BH': 21, 'BI': -12, 'BK': -99, 'BN': 142, 'BO': -56, 'OH': -95,
+ 'OI': 477, 'OK': 410, 'OO': -2422}
UQ2__ = {'BH': 216, 'BI': 113, 'OK': 1759}
UQ3__ = {'BA': -479, 'BH': 42, 'BI': 1913, 'BK': -7198, 'BM': 3160,
- 'BN': 6427, 'BO': 14761, 'OI': -827, 'ON': -3212}
+ 'BN': 6427, 'BO': 14761, 'OI': -827, 'ON': -3212}
UW1__ = {',': 156, '、': 156, '「': -463, 'あ': -941, 'う': -127, 'が': -553,
- 'き': 121, 'こ': 505, 'で': -201, 'と': -547, 'ど': -123, 'に': -789, 'の':
- -185, 'は': -847, 'も': -466, 'や': -470, 'よ': 182, 'ら': -292, 'り':
- 208, 'れ': 169, 'を': -446, 'ん': -137, '・': -135, '主': -402, '京': -
- 268, '区': -912, '午': 871, '国': -460, '大': 561, '委': 729, '市': -411,
- '日': -141, '理': 361, '生': -408, '県': -386, '都': -718, '｢': -463,
- '･': -135}
- UW2__ = {',': -829, '、': -829, '〇': 892, '「': -645, '」': 3145, 'あ': -
- 538, 'い': 505, 'う': 134, 'お': -502, 'か': 1454, 'が': -856, 'く': -412,
- 'こ': 1141, 'さ': 878, 'ざ': 540, 'し': 1529, 'す': -675, 'せ': 300, 'そ':
- -1011, 'た': 188, 'だ': 1837, 'つ': -949, 'て': -291, 'で': -268, 'と': -
- 981, 'ど': 1273, 'な': 1063, 'に': -1764, 'の': 130, 'は': -409, 'ひ': -
- 1273, 'べ': 1261, 'ま': 600, 'も': -1263, 'や': -402, 'よ': 1639, 'り': -
- 579, 'る': -694, 'れ': 571, 'を': -2516, 'ん': 2095, 'ア': -587, 'カ':
- 306, 'キ': 568, 'ッ': 831, '三': -758, '不': -2150, '世': -302, '中': -
- 968, '主': -861, '事': 492, '人': -123, '会': 978, '保': 362, '入': 548,
- '初': -3025, '副': -1566, '北': -3414, '区': -422, '大': -1769, '天': -
- 865, '太': -483, '子': -1519, '学': 760, '実': 1023, '小': -2009, '市': -
- 813, '年': -1060, '強': 1067, '手': -1519, '揺': -1033, '政': 1522, '文':
- -1355, '新': -1682, '日': -1815, '明': -1462, '最': -630, '朝': -1843,
- '本': -1650, '東': -931, '果': -665, '次': -2378, '民': -180, '気': -1740,
- '理': 752, '発': 529, '目': -1584, '相': -242, '県': -1165, '立': -763,
- '第': 810, '米': 509, '自': -1353, '行': 838, '西': -744, '見': -3874,
- '調': 1010, '議': 1198, '込': 3041, '開': 1758, '間': -1257, '｢': -645,
- '｣': 3145, 'ｯ': 831, 'ｱ': -587, 'ｶ': 306, 'ｷ': 568}
- UW3__ = {',': 4889, '1': -800, '−': -1723, '、': 4889, '々': -2311, '〇':
- 5827, '」': 2670, '〓': -3573, 'あ': -2696, 'い': 1006, 'う': 2342, 'え':
- 1983, 'お': -4864, 'か': -1163, 'が': 3271, 'く': 1004, 'け': 388, 'げ':
- 401, 'こ': -3552, 'ご': -3116, 'さ': -1058, 'し': -395, 'す': 584, 'せ':
- 3685, 'そ': -5228, 'た': 842, 'ち': -521, 'っ': -1444, 'つ': -1081, 'て':
- 6167, 'で': 2318, 'と': 1691, 'ど': -899, 'な': -2788, 'に': 2745, 'の':
- 4056, 'は': 4555, 'ひ': -2171, 'ふ': -1798, 'へ': 1199, 'ほ': -5516, 'ま':
- -4384, 'み': -120, 'め': 1205, 'も': 2323, 'や': -788, 'よ': -202, 'ら':
- 727, 'り': 649, 'る': 5905, 'れ': 2773, 'わ': -1207, 'を': 6620, 'ん': -
- 518, 'ア': 551, 'グ': 1319, 'ス': 874, 'ッ': -1350, 'ト': 521, 'ム': 1109,
- 'ル': 1591, 'ロ': 2201, 'ン': 278, '・': -3794, '一': -1619, '下': -1759,
- '世': -2087, '両': 3815, '中': 653, '主': -758, '予': -1193, '二': 974,
- '人': 2742, '今': 792, '他': 1889, '以': -1368, '低': 811, '何': 4265,
- '作': -361, '保': -2439, '元': 4858, '党': 3593, '全': 1574, '公': -3030,
- '六': 755, '共': -1880, '円': 5807, '再': 3095, '分': 457, '初': 2475,
- '別': 1129, '前': 2286, '副': 4437, '力': 365, '動': -949, '務': -1872,
- '化': 1327, '北': -1038, '区': 4646, '千': -2309, '午': -783, '協': -1006,
- '口': 483, '右': 1233, '各': 3588, '合': -241, '同': 3906, '和': -837,
- '員': 4513, '国': 642, '型': 1389, '場': 1219, '外': -241, '妻': 2016,
- '学': -1356, '安': -423, '実': -1008, '家': 1078, '小': -513, '少': -3102,
- '州': 1155, '市': 3197, '平': -1804, '年': 2416, '広': -1030, '府': 1605,
- '度': 1452, '建': -2352, '当': -3885, '得': 1905, '思': -1291, '性': 1822,
- '戸': -488, '指': -3973, '政': -2013, '教': -1479, '数': 3222, '文': -
- 1489, '新': 1764, '日': 2099, '旧': 5792, '昨': -661, '時': -1248, '曜':
- -951, '最': -937, '月': 4125, '期': 360, '李': 3094, '村': 364, '東': -
- 805, '核': 5156, '森': 2438, '業': 484, '氏': 2613, '民': -1694, '決': -
- 1073, '法': 1868, '海': -495, '無': 979, '物': 461, '特': -3850, '生': -
- 273, '用': 914, '町': 1215, '的': 7313, '直': -1835, '省': 792, '県':
- 6293, '知': -1528, '私': 4231, '税': 401, '立': -960, '第': 1201, '米':
- 7767, '系': 3066, '約': 3663, '級': 1384, '統': -4229, '総': 1163, '線':
- 1255, '者': 6457, '能': 725, '自': -2869, '英': 785, '見': 1044, '調': -
- 562, '財': -733, '費': 1777, '車': 1835, '軍': 1375, '込': -1504, '通': -
- 1136, '選': -681, '郎': 1026, '郡': 4404, '部': 1200, '金': 2163, '長':
- 421, '開': -1432, '間': 1302, '関': -1282, '雨': 2009, '電': -1045, '非':
- 2066, '駅': 1620, '１': -800, '｣': 2670, '･': -3794, 'ｯ': -1350, 'ｱ':
- 551, 'ｸﾞ': 1319, 'ｽ': 874, 'ﾄ': 521, 'ﾑ': 1109, 'ﾙ': 1591, 'ﾛ':
- 2201, 'ﾝ': 278}
- UW4__ = {',': 3930, '.': 3508, '―': -4841, '、': 3930, '。': 3508, '〇':
- 4999, '「': 1895, '」': 3798, '〓': -5156, 'あ': 4752, 'い': -3435, 'う':
- -640, 'え': -2514, 'お': 2405, 'か': 530, 'が': 6006, 'き': -4482, 'ぎ':
- -3821, 'く': -3788, 'け': -4376, 'げ': -4734, 'こ': 2255, 'ご': 1979,
- 'さ': 2864, 'し': -843, 'じ': -2506, 'す': -731, 'ず': 1251, 'せ': 181,
- 'そ': 4091, 'た': 5034, 'だ': 5408, 'ち': -3654, 'っ': -5882, 'つ': -1659,
- 'て': 3994, 'で': 7410, 'と': 4547, 'な': 5433, 'に': 6499, 'ぬ': 1853,
- 'ね': 1413, 'の': 7396, 'は': 8578, 'ば': 1940, 'ひ': 4249, 'び': -4134,
- 'ふ': 1345, 'へ': 6665, 'べ': -744, 'ほ': 1464, 'ま': 1051, 'み': -2082,
- 'む': -882, 'め': -5046, 'も': 4169, 'ゃ': -2666, 'や': 2795, 'ょ': -1544,
- 'よ': 3351, 'ら': -2922, 'り': -9726, 'る': -14896, 'れ': -2613, 'ろ': -
- 4570, 'わ': -1783, 'を': 13150, 'ん': -2352, 'カ': 2145, 'コ': 1789, 'セ':
- 1287, 'ッ': -724, 'ト': -403, 'メ': -1635, 'ラ': -881, 'リ': -541, 'ル':
- -856, 'ン': -3637, '・': -4371, 'ー': -11870, '一': -2069, '中': 2210,
- '予': 782, '事': -190, '井': -1768, '人': 1036, '以': 544, '会': 950, '体':
- -1286, '作': 530, '側': 4292, '先': 601, '党': -2006, '共': -1212, '内':
- 584, '円': 788, '初': 1347, '前': 1623, '副': 3879, '力': -302, '動': -
- 740, '務': -2715, '化': 776, '区': 4517, '協': 1013, '参': 1555, '合': -
- 1834, '和': -681, '員': -910, '器': -851, '回': 1500, '国': -619, '園': -
- 1200, '地': 866, '場': -1410, '塁': -2094, '士': -1413, '多': 1067, '大':
- 571, '子': -4802, '学': -1397, '定': -1057, '寺': -809, '小': 1910, '屋':
- -1328, '山': -1500, '島': -2056, '川': -2667, '市': 2771, '年': 374, '庁':
- -4556, '後': 456, '性': 553, '感': 916, '所': -1566, '支': 856, '改': 787,
- '政': 2182, '教': 704, '文': 522, '方': -856, '日': 1798, '時': 1829, '最':
- 845, '月': -9066, '木': -485, '来': -442, '校': -360, '業': -1043, '氏':
- 5388, '民': -2716, '気': -910, '沢': -939, '済': -543, '物': -735, '率':
- 672, '球': -1267, '生': -1286, '産': -1101, '田': -2900, '町': 1826, '的':
- 2586, '目': 922, '省': -3485, '県': 2997, '空': -867, '立': -2112, '第':
- 788, '米': 2937, '系': 786, '約': 2171, '経': 1146, '統': -1169, '総':
- 940, '線': -994, '署': 749, '者': 2145, '能': -730, '般': -852, '行': -
- 792, '規': 792, '警': -1184, '議': -244, '谷': -1000, '賞': 730, '車': -
- 1481, '軍': 1158, '輪': -1433, '込': -3370, '近': 929, '道': -1291, '選':
- 2596, '郎': -4866, '都': 1192, '野': -1100, '銀': -2213, '長': 357, '間':
- -2344, '院': -2297, '際': -2604, '電': -878, '領': -1659, '題': -792,
- '館': -1984, '首': 1749, '高': 2120, '｢': 1895, '｣': 3798, '･': -4371,
- 'ｯ': -724, 'ｰ': -11870, 'ｶ': 2145, 'ｺ': 1789, 'ｾ': 1287, 'ﾄ': -403,
- 'ﾒ': -1635, 'ﾗ': -881, 'ﾘ': -541, 'ﾙ': -856, 'ﾝ': -3637}
- UW5__ = {',': 465, '.': -299, '1': -514, 'E2': -32768, ']': -2762, '、':
- 465, '。': -299, '「': 363, 'あ': 1655, 'い': 331, 'う': -503, 'え': 1199,
- 'お': 527, 'か': 647, 'が': -421, 'き': 1624, 'ぎ': 1971, 'く': 312, 'げ':
- -983, 'さ': -1537, 'し': -1371, 'す': -852, 'だ': -1186, 'ち': 1093, 'っ':
- 52, 'つ': 921, 'て': -18, 'で': -850, 'と': -127, 'ど': 1682, 'な': -787,
- 'に': -1224, 'の': -635, 'は': -578, 'べ': 1001, 'み': 502, 'め': 865,
- 'ゃ': 3350, 'ょ': 854, 'り': -208, 'る': 429, 'れ': 504, 'わ': 419, 'を':
- -1264, 'ん': 327, 'イ': 241, 'ル': 451, 'ン': -343, '中': -871, '京': 722,
- '会': -1153, '党': -654, '務': 3519, '区': -901, '告': 848, '員': 2104,
- '大': -1296, '学': -548, '定': 1785, '嵐': -1304, '市': -2991, '席': 921,
- '年': 1763, '思': 872, '所': -814, '挙': 1618, '新': -1682, '日': 218,
- '月': -4353, '査': 932, '格': 1356, '機': -1508, '氏': -1347, '田': 240,
- '町': -3912, '的': -3149, '相': 1319, '省': -1052, '県': -4003, '研': -
- 997, '社': -278, '空': -813, '統': 1955, '者': -2233, '表': 663, '語': -
- 1073, '議': 1219, '選': -1018, '郎': -368, '長': 786, '間': 1191, '題':
- 2368, '館': -689, '１': -514, 'Ｅ２': -32768, '｢': 363, 'ｲ': 241, 'ﾙ':
- 451, 'ﾝ': -343}
+ 'き': 121, 'こ': 505, 'で': -201, 'と': -547, 'ど': -123, 'に': -789,
+ 'の': -185, 'は': -847, 'も': -466, 'や': -470, 'よ': 182, 'ら': -292,
+ 'り': 208, 'れ': 169, 'を': -446, 'ん': -137, '・': -135, '主': -402,
+ '京': -268, '区': -912, '午': 871, '国': -460, '大': 561, '委': 729,
+ '市': -411, '日': -141, '理': 361, '生': -408, '県': -386, '都': -718,
+ '｢': -463, '･': -135}
+ UW2__ = {',': -829, '、': -829, '〇': 892, '「': -645, '」': 3145, 'あ': -538,
+ 'い': 505, 'う': 134, 'お': -502, 'か': 1454, 'が': -856, 'く': -412,
+ 'こ': 1141, 'さ': 878, 'ざ': 540, 'し': 1529, 'す': -675, 'せ': 300,
+ 'そ': -1011, 'た': 188, 'だ': 1837, 'つ': -949, 'て': -291, 'で': -268,
+ 'と': -981, 'ど': 1273, 'な': 1063, 'に': -1764, 'の': 130, 'は': -409,
+ 'ひ': -1273, 'べ': 1261, 'ま': 600, 'も': -1263, 'や': -402, 'よ': 1639,
+ 'り': -579, 'る': -694, 'れ': 571, 'を': -2516, 'ん': 2095, 'ア': -587,
+ 'カ': 306, 'キ': 568, 'ッ': 831, '三': -758, '不': -2150, '世': -302,
+ '中': -968, '主': -861, '事': 492, '人': -123, '会': 978, '保': 362,
+ '入': 548, '初': -3025, '副': -1566, '北': -3414, '区': -422, '大': -1769,
+ '天': -865, '太': -483, '子': -1519, '学': 760, '実': 1023, '小': -2009,
+ '市': -813, '年': -1060, '強': 1067, '手': -1519, '揺': -1033, '政': 1522,
+ '文': -1355, '新': -1682, '日': -1815, '明': -1462, '最': -630, '朝': -1843,
+ '本': -1650, '東': -931, '果': -665, '次': -2378, '民': -180, '気': -1740,
+ '理': 752, '発': 529, '目': -1584, '相': -242, '県': -1165, '立': -763,
+ '第': 810, '米': 509, '自': -1353, '行': 838, '西': -744, '見': -3874,
+ '調': 1010, '議': 1198, '込': 3041, '開': 1758, '間': -1257, '｢': -645,
+ '｣': 3145, 'ｯ': 831, 'ｱ': -587, 'ｶ': 306, 'ｷ': 568}
+ UW3__ = {',': 4889, '1': -800, '−': -1723, '、': 4889, '々': -2311, '〇': 5827,
+ '」': 2670, '〓': -3573, 'あ': -2696, 'い': 1006, 'う': 2342, 'え': 1983,
+ 'お': -4864, 'か': -1163, 'が': 3271, 'く': 1004, 'け': 388, 'げ': 401,
+ 'こ': -3552, 'ご': -3116, 'さ': -1058, 'し': -395, 'す': 584, 'せ': 3685,
+ 'そ': -5228, 'た': 842, 'ち': -521, 'っ': -1444, 'つ': -1081, 'て': 6167,
+ 'で': 2318, 'と': 1691, 'ど': -899, 'な': -2788, 'に': 2745, 'の': 4056,
+ 'は': 4555, 'ひ': -2171, 'ふ': -1798, 'へ': 1199, 'ほ': -5516, 'ま': -4384,
+ 'み': -120, 'め': 1205, 'も': 2323, 'や': -788, 'よ': -202, 'ら': 727,
+ 'り': 649, 'る': 5905, 'れ': 2773, 'わ': -1207, 'を': 6620, 'ん': -518,
+ 'ア': 551, 'グ': 1319, 'ス': 874, 'ッ': -1350, 'ト': 521, 'ム': 1109,
+ 'ル': 1591, 'ロ': 2201, 'ン': 278, '・': -3794, '一': -1619, '下': -1759,
+ '世': -2087, '両': 3815, '中': 653, '主': -758, '予': -1193, '二': 974,
+ '人': 2742, '今': 792, '他': 1889, '以': -1368, '低': 811, '何': 4265,
+ '作': -361, '保': -2439, '元': 4858, '党': 3593, '全': 1574, '公': -3030,
+ '六': 755, '共': -1880, '円': 5807, '再': 3095, '分': 457, '初': 2475,
+ '別': 1129, '前': 2286, '副': 4437, '力': 365, '動': -949, '務': -1872,
+ '化': 1327, '北': -1038, '区': 4646, '千': -2309, '午': -783, '協': -1006,
+ '口': 483, '右': 1233, '各': 3588, '合': -241, '同': 3906, '和': -837,
+ '員': 4513, '国': 642, '型': 1389, '場': 1219, '外': -241, '妻': 2016,
+ '学': -1356, '安': -423, '実': -1008, '家': 1078, '小': -513, '少': -3102,
+ '州': 1155, '市': 3197, '平': -1804, '年': 2416, '広': -1030, '府': 1605,
+ '度': 1452, '建': -2352, '当': -3885, '得': 1905, '思': -1291, '性': 1822,
+ '戸': -488, '指': -3973, '政': -2013, '教': -1479, '数': 3222, '文': -1489,
+ '新': 1764, '日': 2099, '旧': 5792, '昨': -661, '時': -1248, '曜': -951,
+ '最': -937, '月': 4125, '期': 360, '李': 3094, '村': 364, '東': -805,
+ '核': 5156, '森': 2438, '業': 484, '氏': 2613, '民': -1694, '決': -1073,
+ '法': 1868, '海': -495, '無': 979, '物': 461, '特': -3850, '生': -273,
+ '用': 914, '町': 1215, '的': 7313, '直': -1835, '省': 792, '県': 6293,
+ '知': -1528, '私': 4231, '税': 401, '立': -960, '第': 1201, '米': 7767,
+ '系': 3066, '約': 3663, '級': 1384, '統': -4229, '総': 1163, '線': 1255,
+ '者': 6457, '能': 725, '自': -2869, '英': 785, '見': 1044, '調': -562,
+ '財': -733, '費': 1777, '車': 1835, '軍': 1375, '込': -1504, '通': -1136,
+ '選': -681, '郎': 1026, '郡': 4404, '部': 1200, '金': 2163, '長': 421,
+ '開': -1432, '間': 1302, '関': -1282, '雨': 2009, '電': -1045, '非': 2066,
+ '駅': 1620, '１': -800, '｣': 2670, '･': -3794, 'ｯ': -1350, 'ｱ': 551,
+ 'ｸﾞ': 1319, 'ｽ': 874, 'ﾄ': 521, 'ﾑ': 1109, 'ﾙ': 1591, 'ﾛ': 2201, 'ﾝ': 278}
+ UW4__ = {',': 3930, '.': 3508, '―': -4841, '、': 3930, '。': 3508, '〇': 4999,
+ '「': 1895, '」': 3798, '〓': -5156, 'あ': 4752, 'い': -3435, 'う': -640,
+ 'え': -2514, 'お': 2405, 'か': 530, 'が': 6006, 'き': -4482, 'ぎ': -3821,
+ 'く': -3788, 'け': -4376, 'げ': -4734, 'こ': 2255, 'ご': 1979, 'さ': 2864,
+ 'し': -843, 'じ': -2506, 'す': -731, 'ず': 1251, 'せ': 181, 'そ': 4091,
+ 'た': 5034, 'だ': 5408, 'ち': -3654, 'っ': -5882, 'つ': -1659, 'て': 3994,
+ 'で': 7410, 'と': 4547, 'な': 5433, 'に': 6499, 'ぬ': 1853, 'ね': 1413,
+ 'の': 7396, 'は': 8578, 'ば': 1940, 'ひ': 4249, 'び': -4134, 'ふ': 1345,
+ 'へ': 6665, 'べ': -744, 'ほ': 1464, 'ま': 1051, 'み': -2082, 'む': -882,
+ 'め': -5046, 'も': 4169, 'ゃ': -2666, 'や': 2795, 'ょ': -1544, 'よ': 3351,
+ 'ら': -2922, 'り': -9726, 'る': -14896, 'れ': -2613, 'ろ': -4570,
+ 'わ': -1783, 'を': 13150, 'ん': -2352, 'カ': 2145, 'コ': 1789, 'セ': 1287,
+ 'ッ': -724, 'ト': -403, 'メ': -1635, 'ラ': -881, 'リ': -541, 'ル': -856,
+ 'ン': -3637, '・': -4371, 'ー': -11870, '一': -2069, '中': 2210, '予': 782,
+ '事': -190, '井': -1768, '人': 1036, '以': 544, '会': 950, '体': -1286,
+ '作': 530, '側': 4292, '先': 601, '党': -2006, '共': -1212, '内': 584,
+ '円': 788, '初': 1347, '前': 1623, '副': 3879, '力': -302, '動': -740,
+ '務': -2715, '化': 776, '区': 4517, '協': 1013, '参': 1555, '合': -1834,
+ '和': -681, '員': -910, '器': -851, '回': 1500, '国': -619, '園': -1200,
+ '地': 866, '場': -1410, '塁': -2094, '士': -1413, '多': 1067, '大': 571,
+ '子': -4802, '学': -1397, '定': -1057, '寺': -809, '小': 1910, '屋': -1328,
+ '山': -1500, '島': -2056, '川': -2667, '市': 2771, '年': 374, '庁': -4556,
+ '後': 456, '性': 553, '感': 916, '所': -1566, '支': 856, '改': 787,
+ '政': 2182, '教': 704, '文': 522, '方': -856, '日': 1798, '時': 1829,
+ '最': 845, '月': -9066, '木': -485, '来': -442, '校': -360, '業': -1043,
+ '氏': 5388, '民': -2716, '気': -910, '沢': -939, '済': -543, '物': -735,
+ '率': 672, '球': -1267, '生': -1286, '産': -1101, '田': -2900, '町': 1826,
+ '的': 2586, '目': 922, '省': -3485, '県': 2997, '空': -867, '立': -2112,
+ '第': 788, '米': 2937, '系': 786, '約': 2171, '経': 1146, '統': -1169,
+ '総': 940, '線': -994, '署': 749, '者': 2145, '能': -730, '般': -852,
+ '行': -792, '規': 792, '警': -1184, '議': -244, '谷': -1000, '賞': 730,
+ '車': -1481, '軍': 1158, '輪': -1433, '込': -3370, '近': 929, '道': -1291,
+ '選': 2596, '郎': -4866, '都': 1192, '野': -1100, '銀': -2213, '長': 357,
+ '間': -2344, '院': -2297, '際': -2604, '電': -878, '領': -1659, '題': -792,
+ '館': -1984, '首': 1749, '高': 2120, '｢': 1895, '｣': 3798, '･': -4371,
+ 'ｯ': -724, 'ｰ': -11870, 'ｶ': 2145, 'ｺ': 1789, 'ｾ': 1287, 'ﾄ': -403,
+ 'ﾒ': -1635, 'ﾗ': -881, 'ﾘ': -541, 'ﾙ': -856, 'ﾝ': -3637}
+ UW5__ = {',': 465, '.': -299, '1': -514, 'E2': -32768, ']': -2762, '、': 465,
+ '。': -299, '「': 363, 'あ': 1655, 'い': 331, 'う': -503, 'え': 1199,
+ 'お': 527, 'か': 647, 'が': -421, 'き': 1624, 'ぎ': 1971, 'く': 312,
+ 'げ': -983, 'さ': -1537, 'し': -1371, 'す': -852, 'だ': -1186, 'ち': 1093,
+ 'っ': 52, 'つ': 921, 'て': -18, 'で': -850, 'と': -127, 'ど': 1682,
+ 'な': -787, 'に': -1224, 'の': -635, 'は': -578, 'べ': 1001, 'み': 502,
+ 'め': 865, 'ゃ': 3350, 'ょ': 854, 'り': -208, 'る': 429, 'れ': 504,
+ 'わ': 419, 'を': -1264, 'ん': 327, 'イ': 241, 'ル': 451, 'ン': -343,
+ '中': -871, '京': 722, '会': -1153, '党': -654, '務': 3519, '区': -901,
+ '告': 848, '員': 2104, '大': -1296, '学': -548, '定': 1785, '嵐': -1304,
+ '市': -2991, '席': 921, '年': 1763, '思': 872, '所': -814, '挙': 1618,
+ '新': -1682, '日': 218, '月': -4353, '査': 932, '格': 1356, '機': -1508,
+ '氏': -1347, '田': 240, '町': -3912, '的': -3149, '相': 1319, '省': -1052,
+ '県': -4003, '研': -997, '社': -278, '空': -813, '統': 1955, '者': -2233,
+ '表': 663, '語': -1073, '議': 1219, '選': -1018, '郎': -368, '長': 786,
+ '間': 1191, '題': 2368, '館': -689, '１': -514, 'Ｅ２': -32768, '｢': 363,
+ 'ｲ': 241, 'ﾙ': 451, 'ﾝ': -343}
UW6__ = {',': 227, '.': 808, '1': -270, 'E1': 306, '、': 227, '。': 808,
- 'あ': -307, 'う': 189, 'か': 241, 'が': -73, 'く': -121, 'こ': -200, 'じ':
- 1782, 'す': 383, 'た': -428, 'っ': 573, 'て': -1014, 'で': 101, 'と': -
- 105, 'な': -253, 'に': -149, 'の': -417, 'は': -236, 'も': -206, 'り':
- 187, 'る': -135, 'を': 195, 'ル': -673, 'ン': -496, '一': -277, '中': 201,
- '件': -800, '会': 624, '前': 302, '区': 1792, '員': -1212, '委': 798, '学':
- -960, '市': 887, '広': -695, '後': 535, '業': -697, '相': 753, '社': -507,
- '福': 974, '空': -822, '者': 1811, '連': 463, '郎': 1082, '１': -270,
- 'Ｅ１': 306, 'ﾙ': -673, 'ﾝ': -496}
+ 'あ': -307, 'う': 189, 'か': 241, 'が': -73, 'く': -121, 'こ': -200,
+ 'じ': 1782, 'す': 383, 'た': -428, 'っ': 573, 'て': -1014, 'で': 101,
+ 'と': -105, 'な': -253, 'に': -149, 'の': -417, 'は': -236, 'も': -206,
+ 'り': 187, 'る': -135, 'を': 195, 'ル': -673, 'ン': -496, '一': -277,
+ '中': 201, '件': -800, '会': 624, '前': 302, '区': 1792, '員': -1212,
+ '委': 798, '学': -960, '市': 887, '広': -695, '後': 535, '業': -697,
+ '相': 753, '社': -507, '福': 974, '空': -822, '者': 1811, '連': 463,
+ '郎': 1082, '１': -270, 'Ｅ１': 306, 'ﾙ': -673, 'ﾝ': -496}
+
+ # ctype_
+ def ctype_(self, char: str) -> str:
+ for pattern, value in self.patterns_.items():
+ if pattern.match(char):
+ return value
+ return 'O'
+
+ # ts_
+ def ts_(self, dict: dict[str, int], key: str) -> int:
+ if key in dict:
+ return dict[key]
+ return 0
+
+ # segment
+ def split(self, input: str) -> list[str]:
+ if not input:
+ return []
+
+ result = []
+ seg = ['B3', 'B2', 'B1', *input, 'E1', 'E2', 'E3']
+ ctype = ['O', 'O', 'O', *map(self.ctype_, input), 'O', 'O', 'O']
+ word = seg[3]
+ p1 = 'U'
+ p2 = 'U'
+ p3 = 'U'
+
+ for i in range(4, len(seg) - 3):
+ score = self.BIAS__
+ w1 = seg[i-3]
+ w2 = seg[i-2]
+ w3 = seg[i-1]
+ w4 = seg[i]
+ w5 = seg[i+1]
+ w6 = seg[i+2]
+ c1 = ctype[i-3]
+ c2 = ctype[i-2]
+ c3 = ctype[i-1]
+ c4 = ctype[i]
+ c5 = ctype[i+1]
+ c6 = ctype[i+2]
+ score += self.ts_(self.UP1__, p1)
+ score += self.ts_(self.UP2__, p2)
+ score += self.ts_(self.UP3__, p3)
+ score += self.ts_(self.BP1__, p1 + p2)
+ score += self.ts_(self.BP2__, p2 + p3)
+ score += self.ts_(self.UW1__, w1)
+ score += self.ts_(self.UW2__, w2)
+ score += self.ts_(self.UW3__, w3)
+ score += self.ts_(self.UW4__, w4)
+ score += self.ts_(self.UW5__, w5)
+ score += self.ts_(self.UW6__, w6)
+ score += self.ts_(self.BW1__, w2 + w3)
+ score += self.ts_(self.BW2__, w3 + w4)
+ score += self.ts_(self.BW3__, w4 + w5)
+ score += self.ts_(self.TW1__, w1 + w2 + w3)
+ score += self.ts_(self.TW2__, w2 + w3 + w4)
+ score += self.ts_(self.TW3__, w3 + w4 + w5)
+ score += self.ts_(self.TW4__, w4 + w5 + w6)
+ score += self.ts_(self.UC1__, c1)
+ score += self.ts_(self.UC2__, c2)
+ score += self.ts_(self.UC3__, c3)
+ score += self.ts_(self.UC4__, c4)
+ score += self.ts_(self.UC5__, c5)
+ score += self.ts_(self.UC6__, c6)
+ score += self.ts_(self.BC1__, c2 + c3)
+ score += self.ts_(self.BC2__, c3 + c4)
+ score += self.ts_(self.BC3__, c4 + c5)
+ score += self.ts_(self.TC1__, c1 + c2 + c3)
+ score += self.ts_(self.TC2__, c2 + c3 + c4)
+ score += self.ts_(self.TC3__, c3 + c4 + c5)
+ score += self.ts_(self.TC4__, c4 + c5 + c6)
+# score += self.ts_(self.TC5__, c4 + c5 + c6)
+ score += self.ts_(self.UQ1__, p1 + c1)
+ score += self.ts_(self.UQ2__, p2 + c2)
+ score += self.ts_(self.UQ1__, p3 + c3)
+ score += self.ts_(self.BQ1__, p2 + c2 + c3)
+ score += self.ts_(self.BQ2__, p2 + c3 + c4)
+ score += self.ts_(self.BQ3__, p3 + c2 + c3)
+ score += self.ts_(self.BQ4__, p3 + c3 + c4)
+ score += self.ts_(self.TQ1__, p2 + c1 + c2 + c3)
+ score += self.ts_(self.TQ2__, p2 + c2 + c3 + c4)
+ score += self.ts_(self.TQ3__, p3 + c1 + c2 + c3)
+ score += self.ts_(self.TQ4__, p3 + c2 + c3 + c4)
+ p = 'O'
+ if score > 0:
+ result.append(word.strip())
+ word = ''
+ p = 'B'
+ p1 = p2
+ p2 = p3
+ p3 = p
+ word += seg[i]
+
+ result.append(word.strip())
+ return result
class SearchJapanese(SearchLanguage):
@@ -330,3 +503,26 @@ class SearchJapanese(SearchLanguage):
"""
lang = 'ja'
language_name = 'Japanese'
+
+ def init(self, options: dict[str, str]) -> None:
+ dotted_path = options.get('type')
+ if dotted_path is None:
+ self.splitter = DefaultSplitter(options)
+ else:
+ try:
+ splitter_cls = import_object(
+ dotted_path, "html_search_options['type'] setting"
+ )
+ self.splitter = splitter_cls(options)
+ except ExtensionError as exc:
+ msg = f"Splitter module {dotted_path!r} can't be imported"
+ raise ExtensionError(msg) from exc
+
+ def split(self, input: str) -> list[str]:
+ return self.splitter.split(input)
+
+ def word_filter(self, stemmed_word: str) -> bool:
+ return len(stemmed_word) > 1
+
+ def stem(self, word: str) -> str:
+ return word
diff --git a/sphinx/search/nl.py b/sphinx/search/nl.py
index 08808edf6..cb5e8c4f9 100644
--- a/sphinx/search/nl.py
+++ b/sphinx/search/nl.py
@@ -1,10 +1,14 @@
"""Dutch search language: includes the JS porter stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-dutch_stopwords = parse_stop_word(
- """
+
+dutch_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/dutch/stop.txt
de | the
en | and
@@ -107,8 +111,7 @@ uw | your
iemand | somebody
geweest | been; past participle of 'be'
andere | other
-"""
- )
+''')
class SearchDutch(SearchLanguage):
@@ -116,3 +119,9 @@ class SearchDutch(SearchLanguage):
language_name = 'Dutch'
js_stemmer_rawcode = 'dutch-stemmer.js'
stopwords = dutch_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('dutch')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/no.py b/sphinx/search/no.py
index eb6e4e26d..aa7c1043b 100644
--- a/sphinx/search/no.py
+++ b/sphinx/search/no.py
@@ -1,10 +1,14 @@
"""Norwegian search language: includes the JS Norwegian stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-norwegian_stopwords = parse_stop_word(
- """
+
+norwegian_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/norwegian/stop.txt
og | and
i | in
@@ -182,8 +186,7 @@ verte | become *
vort | become *
varte | became *
vart | became *
-"""
- )
+''')
class SearchNorwegian(SearchLanguage):
@@ -191,3 +194,9 @@ class SearchNorwegian(SearchLanguage):
language_name = 'Norwegian'
js_stemmer_rawcode = 'norwegian-stemmer.js'
stopwords = norwegian_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('norwegian')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/pt.py b/sphinx/search/pt.py
index 634958238..0cf96109a 100644
--- a/sphinx/search/pt.py
+++ b/sphinx/search/pt.py
@@ -1,10 +1,14 @@
"""Portuguese search language: includes the JS Portuguese stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-portuguese_stopwords = parse_stop_word(
- """
+
+portuguese_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/portuguese/stop.txt
de | of, from
a | the; to, at; her
@@ -241,8 +245,7 @@ terão
teria
teríamos
teriam
-"""
- )
+''')
class SearchPortuguese(SearchLanguage):
@@ -250,3 +253,9 @@ class SearchPortuguese(SearchLanguage):
language_name = 'Portuguese'
js_stemmer_rawcode = 'portuguese-stemmer.js'
stopwords = portuguese_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('portuguese')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/ro.py b/sphinx/search/ro.py
index 85243751a..f15b7a6bb 100644
--- a/sphinx/search/ro.py
+++ b/sphinx/search/ro.py
@@ -1,7 +1,11 @@
"""Romanian search language: includes the JS Romanian stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict, Set
+
import snowballstemmer
+
from sphinx.search import SearchLanguage
@@ -10,3 +14,9 @@ class SearchRomanian(SearchLanguage):
language_name = 'Romanian'
js_stemmer_rawcode = 'romanian-stemmer.js'
stopwords: set[str] = set()
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('romanian')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/ru.py b/sphinx/search/ru.py
index a2a8a8712..d6b817ebe 100644
--- a/sphinx/search/ru.py
+++ b/sphinx/search/ru.py
@@ -1,10 +1,14 @@
"""Russian search language: includes the JS Russian stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-russian_stopwords = parse_stop_word(
- """
+
+russian_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/russian/stop.txt
и | and
в | in/into
@@ -231,8 +235,7 @@ russian_stopwords = parse_stop_word(
| можн
| нужн
| нельзя
-"""
- )
+''')
class SearchRussian(SearchLanguage):
@@ -240,3 +243,9 @@ class SearchRussian(SearchLanguage):
language_name = 'Russian'
js_stemmer_rawcode = 'russian-stemmer.js'
stopwords = russian_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('russian')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/sv.py b/sphinx/search/sv.py
index 3276ce85a..b90e22764 100644
--- a/sphinx/search/sv.py
+++ b/sphinx/search/sv.py
@@ -1,10 +1,14 @@
"""Swedish search language: includes the JS Swedish stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict
+
import snowballstemmer
+
from sphinx.search import SearchLanguage, parse_stop_word
-swedish_stopwords = parse_stop_word(
- """
+
+swedish_stopwords = parse_stop_word('''
| source: https://snowball.tartarus.org/algorithms/swedish/stop.txt
och | and
det | it, this/that
@@ -120,8 +124,7 @@ våra | our
ert | your
era | your
vilkas | whose
-"""
- )
+''')
class SearchSwedish(SearchLanguage):
@@ -129,3 +132,9 @@ class SearchSwedish(SearchLanguage):
language_name = 'Swedish'
js_stemmer_rawcode = 'swedish-stemmer.js'
stopwords = swedish_stopwords
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('swedish')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/tr.py b/sphinx/search/tr.py
index 582d9271b..fdfc18a22 100644
--- a/sphinx/search/tr.py
+++ b/sphinx/search/tr.py
@@ -1,7 +1,11 @@
"""Turkish search language: includes the JS Turkish stemmer."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Dict, Set
+
import snowballstemmer
+
from sphinx.search import SearchLanguage
@@ -10,3 +14,9 @@ class SearchTurkish(SearchLanguage):
language_name = 'Turkish'
js_stemmer_rawcode = 'turkish-stemmer.js'
stopwords: set[str] = set()
+
+ def init(self, options: dict[str, str]) -> None:
+ self.stemmer = snowballstemmer.stemmer('turkish')
+
+ def stem(self, word: str) -> str:
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/search/zh.py b/sphinx/search/zh.py
index c10d2f820..e40c9a9fe 100644
--- a/sphinx/search/zh.py
+++ b/sphinx/search/zh.py
@@ -1,16 +1,21 @@
"""Chinese search language: includes routine to split words."""
+
from __future__ import annotations
+
import os
import re
+
import snowballstemmer
+
from sphinx.search import SearchLanguage
+
try:
- import jieba
+ import jieba # type: ignore[import-not-found]
JIEBA = True
except ImportError:
JIEBA = False
-english_stopwords = set(
- """
+
+english_stopwords = set("""
a and are as at
be but by
for
@@ -20,8 +25,8 @@ of on or
such
that the their then there these they this to
was will with
-"""
- .split())
+""".split())
+
js_porter_stemmer = """
/**
* Porter Stemmer
@@ -141,7 +146,8 @@ var Stemmer = function() {
}
// Step 2
- re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;
+ re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|\
+ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;
if (re.test(w)) {
var fp = re.exec(w);
stem = fp[1];
@@ -163,7 +169,8 @@ var Stemmer = function() {
}
// Step 4
- re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/;
+ re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|\
+iti|ous|ive|ize)$/;
re2 = /^(.+?)(s|t)(ion)$/;
if (re.test(w)) {
var fp = re.exec(w);
@@ -211,9 +218,44 @@ class SearchChinese(SearchLanguage):
"""
Chinese search implementation
"""
+
lang = 'zh'
language_name = 'Chinese'
js_stemmer_code = js_porter_stemmer
stopwords = english_stopwords
- latin1_letters = re.compile('[a-zA-Z0-9_]+')
+ latin1_letters = re.compile(r'[a-zA-Z0-9_]+')
latin_terms: list[str] = []
+
+ def init(self, options: dict[str, str]) -> None:
+ if JIEBA:
+ dict_path = options.get('dict')
+ if dict_path and os.path.isfile(dict_path):
+ jieba.load_userdict(dict_path)
+
+ self.stemmer = snowballstemmer.stemmer('english')
+
+ def split(self, input: str) -> list[str]:
+ chinese: list[str] = []
+ if JIEBA:
+ chinese = list(jieba.cut_for_search(input))
+
+ latin1 = \
+ [term.strip() for term in self.latin1_letters.findall(input)]
+ self.latin_terms.extend(latin1)
+ return chinese + latin1
+
+ def word_filter(self, stemmed_word: str) -> bool:
+ return len(stemmed_word) > 1
+
+ def stem(self, word: str) -> str:
+ # Don't stem Latin words that are long enough to be relevant for search
+ # if not stemmed, but would be too short after being stemmed
+ # avoids some issues with acronyms
+ should_not_be_stemmed = (
+ word in self.latin_terms and
+ len(word) >= 3 and
+ len(self.stemmer.stemWord(word.lower())) < 3
+ )
+ if should_not_be_stemmed:
+ return word.lower()
+ return self.stemmer.stemWord(word.lower())
diff --git a/sphinx/testing/fixtures.py b/sphinx/testing/fixtures.py
index 354a465af..03e38e85e 100644
--- a/sphinx/testing/fixtures.py
+++ b/sphinx/testing/fixtures.py
@@ -1,46 +1,124 @@
"""Sphinx test fixtures for pytest"""
+
from __future__ import annotations
+
import shutil
import subprocess
import sys
from collections import namedtuple
from io import StringIO
from typing import TYPE_CHECKING
+
import pytest
+
from sphinx.testing.util import SphinxTestApp, SphinxTestAppWrapperForSkipBuilding
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
from pathlib import Path
from typing import Any
+
DEFAULT_ENABLED_MARKERS = [
- 'sphinx(buildername="html", *, testroot="root", srcdir=None, confoverrides=None, freshenv=False, warningiserror=False, tags=None, verbosity=0, parallel=0, builddir=None, docutils_conf=None): arguments to initialize the sphinx test application.'
- , 'test_params(shared_result=...): test parameters.']
+ # The marker signature differs from the constructor signature
+ # since the way it is processed assumes keyword arguments for
+ # the 'testroot' and 'srcdir'.
+ (
+ 'sphinx('
+ 'buildername="html", *, '
+ 'testroot="root", srcdir=None, '
+ 'confoverrides=None, freshenv=False, '
+ 'warningiserror=False, tags=None, verbosity=0, parallel=0, '
+ 'builddir=None, docutils_conf=None'
+ '): arguments to initialize the sphinx test application.'
+ ),
+ 'test_params(shared_result=...): test parameters.',
+]
-def pytest_configure(config: pytest.Config) ->None:
+def pytest_configure(config: pytest.Config) -> None:
"""Register custom markers"""
- pass
+ for marker in DEFAULT_ENABLED_MARKERS:
+ config.addinivalue_line('markers', marker)
+
+
+@pytest.fixture(scope='session')
+def rootdir() -> Path | None:
+ return None
class SharedResult:
cache: dict[str, dict[str, str]] = {}
+ def store(self, key: str, app_: SphinxTestApp) -> Any:
+ if key in self.cache:
+ return
+ data = {
+ 'status': app_.status.getvalue(),
+ 'warning': app_.warning.getvalue(),
+ }
+ self.cache[key] = data
+
+ def restore(self, key: str) -> dict[str, StringIO]:
+ if key not in self.cache:
+ return {}
+ data = self.cache[key]
+ return {
+ 'status': StringIO(data['status']),
+ 'warning': StringIO(data['warning']),
+ }
+
@pytest.fixture
-def app_params(request: Any, test_params: dict[str, Any], shared_result:
- SharedResult, sphinx_test_tempdir: str, rootdir: Path) ->_app_params:
+def app_params(
+ request: Any,
+ test_params: dict[str, Any],
+ shared_result: SharedResult,
+ sphinx_test_tempdir: str,
+ rootdir: Path,
+) -> _app_params:
"""
Parameters that are specified by 'pytest.mark.sphinx' for
sphinx.application.Sphinx initialization
"""
- pass
+ # ##### process pytest.mark.sphinx
+
+ pargs: dict[int, Any] = {}
+ kwargs: dict[str, Any] = {}
+
+ # to avoid stacking positional args
+ for info in reversed(list(request.node.iter_markers("sphinx"))):
+ pargs |= dict(enumerate(info.args))
+ kwargs.update(info.kwargs)
+
+ args = [pargs[i] for i in sorted(pargs.keys())]
+
+ # ##### process pytest.mark.test_params
+ if test_params['shared_result']:
+ if 'srcdir' in kwargs:
+ msg = 'You can not specify shared_result and srcdir in same time.'
+ pytest.fail(msg)
+ kwargs['srcdir'] = test_params['shared_result']
+ restore = shared_result.restore(test_params['shared_result'])
+ kwargs.update(restore)
+
+ # ##### prepare Application params
+
+ testroot = kwargs.pop('testroot', 'root')
+ kwargs['srcdir'] = srcdir = sphinx_test_tempdir / kwargs.get('srcdir', testroot)
+
+ # special support for sphinx/tests
+ if rootdir and not srcdir.exists():
+ testroot_path = rootdir / ('test-' + testroot)
+ shutil.copytree(testroot_path, srcdir)
+
+ return _app_params(args, kwargs)
_app_params = namedtuple('_app_params', 'args,kwargs')
@pytest.fixture
-def test_params(request: Any) ->dict[str, Any]:
+def test_params(request: Any) -> dict[str, Any]:
"""
Test parameters that are specified by 'pytest.mark.test_params'
@@ -50,63 +128,124 @@ def test_params(request: Any) ->dict[str, Any]:
have same 'shared_result' value.
**NOTE**: You can not specify both shared_result and srcdir.
"""
- pass
+ env = request.node.get_closest_marker('test_params')
+ kwargs = env.kwargs if env else {}
+ result = {
+ 'shared_result': None,
+ }
+ result.update(kwargs)
+
+ if result['shared_result'] and not isinstance(result['shared_result'], str):
+ msg = 'You can only provide a string type of value for "shared_result"'
+ raise pytest.Exception(msg)
+ return result
@pytest.fixture
-def app(test_params: dict[str, Any], app_params: _app_params, make_app:
- Callable[[], SphinxTestApp], shared_result: SharedResult) ->Iterator[
- SphinxTestApp]:
+def app(
+ test_params: dict[str, Any],
+ app_params: _app_params,
+ make_app: Callable[[], SphinxTestApp],
+ shared_result: SharedResult,
+) -> Iterator[SphinxTestApp]:
"""
Provides the 'sphinx.application.Sphinx' object
"""
- pass
+ args, kwargs = app_params
+ app_ = make_app(*args, **kwargs)
+ yield app_
+
+ print('# testroot:', kwargs.get('testroot', 'root'))
+ print('# builder:', app_.builder.name)
+ print('# srcdir:', app_.srcdir)
+ print('# outdir:', app_.outdir)
+ print('# status:', '\n' + app_.status.getvalue())
+ print('# warning:', '\n' + app_.warning.getvalue())
+
+ if test_params['shared_result']:
+ shared_result.store(test_params['shared_result'], app_)
@pytest.fixture
-def status(app: SphinxTestApp) ->StringIO:
+def status(app: SphinxTestApp) -> StringIO:
"""
Back-compatibility for testing with previous @with_app decorator
"""
- pass
+ return app.status
@pytest.fixture
-def warning(app: SphinxTestApp) ->StringIO:
+def warning(app: SphinxTestApp) -> StringIO:
"""
Back-compatibility for testing with previous @with_app decorator
"""
- pass
+ return app.warning
@pytest.fixture
-def make_app(test_params: dict[str, Any]) ->Iterator[Callable[[],
- SphinxTestApp]]:
+def make_app(test_params: dict[str, Any]) -> Iterator[Callable[[], SphinxTestApp]]:
"""
Provides make_app function to initialize SphinxTestApp instance.
if you want to initialize 'app' in your test function. please use this
instead of using SphinxTestApp class directory.
"""
- pass
+ apps = []
+ syspath = sys.path.copy()
+
+ def make(*args: Any, **kwargs: Any) -> SphinxTestApp:
+ status, warning = StringIO(), StringIO()
+ kwargs.setdefault('status', status)
+ kwargs.setdefault('warning', warning)
+ app_: SphinxTestApp
+ if test_params['shared_result']:
+ app_ = SphinxTestAppWrapperForSkipBuilding(*args, **kwargs)
+ else:
+ app_ = SphinxTestApp(*args, **kwargs)
+ apps.append(app_)
+ return app_
+ yield make
+
+ sys.path[:] = syspath
+ for app_ in reversed(apps): # clean up applications from the new ones
+ app_.cleanup()
@pytest.fixture
-def if_graphviz_found(app: SphinxTestApp) ->None:
+def shared_result() -> SharedResult:
+ return SharedResult()
+
+
+@pytest.fixture(scope='module', autouse=True)
+def _shared_result_cache() -> None:
+ SharedResult.cache.clear()
+
+
+@pytest.fixture
+def if_graphviz_found(app: SphinxTestApp) -> None: # NoQA: PT004
"""
The test will be skipped when using 'if_graphviz_found' fixture and graphviz
dot command is not found.
"""
- pass
+ graphviz_dot = getattr(app.config, 'graphviz_dot', '')
+ try:
+ if graphviz_dot:
+ # print the graphviz_dot version, to check that the binary is available
+ subprocess.run([graphviz_dot, '-V'], capture_output=True, check=False)
+ return
+ except OSError: # No such file or directory
+ pass
+
+ pytest.skip('graphviz "dot" is not available')
@pytest.fixture(scope='session')
-def sphinx_test_tempdir(tmp_path_factory: pytest.TempPathFactory) ->Path:
+def sphinx_test_tempdir(tmp_path_factory: pytest.TempPathFactory) -> Path:
"""Temporary directory."""
- pass
+ return tmp_path_factory.getbasetemp()
@pytest.fixture
-def rollback_sysmodules() ->Iterator[None]:
+def rollback_sysmodules() -> Iterator[None]: # NoQA: PT004
"""
Rollback sys.modules to its value before testing to unload modules
during tests.
@@ -114,4 +253,10 @@ def rollback_sysmodules() ->Iterator[None]:
For example, used in test_ext_autosummary.py to permit unloading the
target module to clear its cache.
"""
- pass
+ sysmodules = list(sys.modules)
+ try:
+ yield
+ finally:
+ for modname in list(sys.modules):
+ if modname not in sysmodules:
+ sys.modules.pop(modname)
diff --git a/sphinx/testing/path.py b/sphinx/testing/path.py
index 6aa6b7bf7..49f0ffa60 100644
--- a/sphinx/testing/path.py
+++ b/sphinx/testing/path.py
@@ -1,22 +1,30 @@
from __future__ import annotations
+
import os
import shutil
import sys
import warnings
from typing import IO, TYPE_CHECKING, Any
+
from sphinx.deprecation import RemovedInSphinx90Warning
+
if TYPE_CHECKING:
import builtins
from collections.abc import Callable
-warnings.warn(
- "'sphinx.testing.path' is deprecated. Use 'os.path' or 'pathlib' instead.",
- RemovedInSphinx90Warning, stacklevel=2)
+
+warnings.warn("'sphinx.testing.path' is deprecated. "
+ "Use 'os.path' or 'pathlib' instead.",
+ RemovedInSphinx90Warning, stacklevel=2)
+
FILESYSTEMENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding()
-def getumask() ->int:
+def getumask() -> int:
"""Get current umask value"""
- pass
+ umask = os.umask(0) # Note: Change umask value temporarily to obtain it
+ os.umask(umask)
+
+ return umask
UMASK = getumask()
@@ -26,53 +34,60 @@ class path(str):
"""
Represents a path which behaves like a string.
"""
+
__slots__ = ()
@property
- def parent(self) ->path:
+ def parent(self) -> path:
"""
The name of the directory the file or directory is in.
"""
- pass
+ return self.__class__(os.path.dirname(self))
- def abspath(self) ->path:
+ def basename(self) -> str:
+ return os.path.basename(self)
+
+ def abspath(self) -> path:
"""
Returns the absolute path.
"""
- pass
+ return self.__class__(os.path.abspath(self))
- def isabs(self) ->bool:
+ def isabs(self) -> bool:
"""
Returns ``True`` if the path is absolute.
"""
- pass
+ return os.path.isabs(self)
- def isdir(self) ->bool:
+ def isdir(self) -> bool:
"""
Returns ``True`` if the path is a directory.
"""
- pass
+ return os.path.isdir(self)
- def isfile(self) ->bool:
+ def isfile(self) -> bool:
"""
Returns ``True`` if the path is a file.
"""
- pass
+ return os.path.isfile(self)
- def islink(self) ->bool:
+ def islink(self) -> bool:
"""
Returns ``True`` if the path is a symbolic link.
"""
- pass
+ return os.path.islink(self)
- def ismount(self) ->bool:
+ def ismount(self) -> bool:
"""
Returns ``True`` if the path is a mount point.
"""
- pass
+ return os.path.ismount(self)
- def rmtree(self, ignore_errors: bool=False, onerror: (Callable[[
- Callable[..., Any], str, Any], object] | None)=None) ->None:
+ def rmtree(
+ self,
+ ignore_errors: bool = False,
+ onerror: Callable[[Callable[..., Any], str, Any], object] | None = None,
+ ) -> None:
"""
Removes the file or directory and any files or directories it may
contain.
@@ -88,9 +103,9 @@ class path(str):
caused it to fail and `exc_info` is a tuple as returned by
:func:`sys.exc_info`.
"""
- pass
+ shutil.rmtree(self, ignore_errors=ignore_errors, onerror=onerror)
- def copytree(self, destination: str, symlinks: bool=False) ->None:
+ def copytree(self, destination: str, symlinks: bool = False) -> None:
"""
Recursively copy a directory to the given `destination`. If the given
`destination` does not exist it will be created.
@@ -100,9 +115,19 @@ class path(str):
links in the destination tree otherwise the contents of the files
pointed to by the symbolic links are copied.
"""
- pass
+ shutil.copytree(self, destination, symlinks=symlinks)
+ if os.environ.get('SPHINX_READONLY_TESTDIR'):
+ # If source tree is marked read-only (e.g. because it is on a read-only
+ # filesystem), `shutil.copytree` will mark the destination as read-only
+ # as well. To avoid failures when adding additional files/directories
+ # to the destination tree, ensure destination directories are not marked
+ # read-only.
+ for root, _dirs, files in os.walk(destination):
+ os.chmod(root, 0o755 & ~UMASK)
+ for name in files:
+ os.chmod(os.path.join(root, name), 0o644 & ~UMASK)
- def movetree(self, destination: str) ->None:
+ def movetree(self, destination: str) -> None:
"""
Recursively move the file or directory to the given `destination`
similar to the Unix "mv" command.
@@ -110,74 +135,92 @@ class path(str):
If the `destination` is a file it may be overwritten depending on the
:func:`os.rename` semantics.
"""
- pass
+ shutil.move(self, destination)
+
move = movetree
- def unlink(self) ->None:
+ def unlink(self) -> None:
"""
Removes a file.
"""
- pass
+ os.unlink(self)
- def stat(self) ->Any:
+ def stat(self) -> Any:
"""
Returns a stat of the file.
"""
- pass
+ return os.stat(self)
+
+ def utime(self, arg: Any) -> None:
+ os.utime(self, arg)
- def write_text(self, text: str, encoding: str='utf-8', **kwargs: Any
- ) ->None:
+ def open(self, mode: str = 'r', **kwargs: Any) -> IO[str]:
+ return open(self, mode, **kwargs) # NoQA: SIM115
+
+ def write_text(self, text: str, encoding: str = 'utf-8', **kwargs: Any) -> None:
"""
Writes the given `text` to the file.
"""
- pass
+ with open(self, 'w', encoding=encoding, **kwargs) as f:
+ f.write(text)
- def read_text(self, encoding: str='utf-8', **kwargs: Any) ->str:
+ def read_text(self, encoding: str = 'utf-8', **kwargs: Any) -> str:
"""
Returns the text in the file.
"""
- pass
+ with open(self, encoding=encoding, **kwargs) as f:
+ return f.read()
- def read_bytes(self) ->builtins.bytes:
+ def read_bytes(self) -> builtins.bytes:
"""
Returns the bytes in the file.
"""
- pass
+ with open(self, mode='rb') as f:
+ return f.read()
- def write_bytes(self, bytes: bytes, append: bool=False) ->None:
+ def write_bytes(self, bytes: bytes, append: bool = False) -> None:
"""
Writes the given `bytes` to the file.
:param append:
If ``True`` given `bytes` are added at the end of the file.
"""
- pass
+ if append:
+ mode = 'ab'
+ else:
+ mode = 'wb'
+ with open(self, mode=mode) as f:
+ f.write(bytes)
- def exists(self) ->bool:
+ def exists(self) -> bool:
"""
Returns ``True`` if the path exist.
"""
- pass
+ return os.path.exists(self)
- def lexists(self) ->bool:
+ def lexists(self) -> bool:
"""
Returns ``True`` if the path exists unless it is a broken symbolic
link.
"""
- pass
+ return os.path.lexists(self)
- def makedirs(self, mode: int=511, exist_ok: bool=False) ->None:
+ def makedirs(self, mode: int = 0o777, exist_ok: bool = False) -> None:
"""
Recursively create directories.
"""
- pass
+ os.makedirs(self, mode, exist_ok=exist_ok)
- def joinpath(self, *args: Any) ->path:
+ def joinpath(self, *args: Any) -> path:
"""
Joins the path with the argument given and returns the result.
"""
- pass
+ return self.__class__(os.path.join(self, *map(self.__class__, args)))
+
+ def listdir(self) -> list[str]:
+ return os.listdir(self)
+
__div__ = __truediv__ = joinpath
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return f'{self.__class__.__name__}({super().__repr__()})'
diff --git a/sphinx/testing/restructuredtext.py b/sphinx/testing/restructuredtext.py
index f8e53bc18..1f89336db 100644
--- a/sphinx/testing/restructuredtext.py
+++ b/sphinx/testing/restructuredtext.py
@@ -1,12 +1,35 @@
from os import path
+
from docutils import nodes
from docutils.core import publish_doctree
+
from sphinx.application import Sphinx
from sphinx.io import SphinxStandaloneReader
from sphinx.parsers import RSTParser
from sphinx.util.docutils import sphinx_domains
-def parse(app: Sphinx, text: str, docname: str='index') ->nodes.document:
+def parse(app: Sphinx, text: str, docname: str = 'index') -> nodes.document:
"""Parse a string as reStructuredText with Sphinx application."""
- pass
+ try:
+ app.env.temp_data['docname'] = docname
+ reader = SphinxStandaloneReader()
+ reader.setup(app)
+ parser = RSTParser()
+ parser.set_application(app)
+ with sphinx_domains(app.env):
+ return publish_doctree(
+ text,
+ path.join(app.srcdir, docname + '.rst'),
+ reader=reader,
+ parser=parser,
+ settings_overrides={
+ 'env': app.env,
+ 'gettext_compact': True,
+ 'input_encoding': 'utf-8',
+ 'output_encoding': 'unicode',
+ 'traceback': True,
+ },
+ )
+ finally:
+ app.env.temp_data.pop('docname', None)
diff --git a/sphinx/testing/util.py b/sphinx/testing/util.py
index 6b9e7daa2..9cf77d304 100644
--- a/sphinx/testing/util.py
+++ b/sphinx/testing/util.py
@@ -1,30 +1,80 @@
"""Sphinx test suite utilities"""
+
from __future__ import annotations
-__all__ = 'SphinxTestApp', 'SphinxTestAppWrapperForSkipBuilding'
+
+__all__ = ('SphinxTestApp', 'SphinxTestAppWrapperForSkipBuilding')
+
import contextlib
import os
import sys
from io import StringIO
from types import MappingProxyType
from typing import TYPE_CHECKING
+
from docutils import nodes
from docutils.parsers.rst import directives, roles
+
import sphinx.application
import sphinx.locale
import sphinx.pycode
from sphinx.util.console import strip_colors
from sphinx.util.docutils import additional_nodes
+
if TYPE_CHECKING:
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from xml.etree.ElementTree import ElementTree
+
from docutils.nodes import Node
-def etree_parse(path: (str | os.PathLike[str])) ->ElementTree:
+def assert_node(node: Node, cls: Any = None, xpath: str = "", **kwargs: Any) -> None:
+ if cls:
+ if isinstance(cls, list):
+ assert_node(node, cls[0], xpath=xpath, **kwargs)
+ if cls[1:]:
+ if isinstance(cls[1], tuple):
+ assert_node(node, cls[1], xpath=xpath, **kwargs)
+ else:
+ assert isinstance(node, nodes.Element), \
+ 'The node%s does not have any children' % xpath
+ assert len(node) == 1, \
+ 'The node%s has %d child nodes, not one' % (xpath, len(node))
+ assert_node(node[0], cls[1:], xpath=xpath + "[0]", **kwargs)
+ elif isinstance(cls, tuple):
+ assert isinstance(node, list | nodes.Element), \
+ 'The node%s does not have any items' % xpath
+ assert len(node) == len(cls), \
+ 'The node%s has %d child nodes, not %r' % (xpath, len(node), len(cls))
+ for i, nodecls in enumerate(cls):
+ path = xpath + "[%d]" % i
+ assert_node(node[i], nodecls, xpath=path, **kwargs)
+ elif isinstance(cls, str):
+ assert node == cls, f'The node {xpath!r} is not {cls!r}: {node!r}'
+ else:
+ assert isinstance(node, cls), \
+ f'The node{xpath} is not subclass of {cls!r}: {node!r}'
+
+ if kwargs:
+ assert isinstance(node, nodes.Element), \
+ 'The node%s does not have any attributes' % xpath
+
+ for key, value in kwargs.items():
+ if key not in node:
+ if (key := key.replace('_', '-')) not in node:
+ msg = f'The node{xpath} does not have {key!r} attribute: {node!r}'
+ raise AssertionError(msg)
+ assert node[key] == value, \
+ f'The node{xpath}[{key}] is not {value!r}: {node[key]!r}'
+
+
+# keep this to restrict the API usage and to have a correct return type
+def etree_parse(path: str | os.PathLike[str]) -> ElementTree:
"""Parse a file into a (safe) XML element tree."""
- pass
+ from defusedxml.ElementTree import parse as xml_parse
+
+ return xml_parse(path)
class SphinxTestApp(sphinx.application.Sphinx):
@@ -49,36 +99,61 @@ class SphinxTestApp(sphinx.application.Sphinx):
directory, whereas in the latter, the user must provide it themselves.
"""
- def __init__(self, /, buildername: str='html', srcdir: (Path | None)=
- None, builddir: (Path | None)=None, freshenv: bool=False,
- confoverrides: (dict[str, Any] | None)=None, status: (StringIO |
- None)=None, warning: (StringIO | None)=None, tags: Sequence[str]=(),
- docutils_conf: (str | None)=None, parallel: int=0, verbosity: int=0,
- warningiserror: bool=False, pdb: bool=False, exception_on_warning:
- bool=False, **extras: Any) ->None:
+ # see https://github.com/sphinx-doc/sphinx/pull/12089 for the
+ # discussion on how the signature of this class should be used
+
+ def __init__(
+ self,
+ /, # to allow 'self' as an extras
+ buildername: str = 'html',
+ srcdir: Path | None = None,
+ builddir: Path | None = None, # extra constructor argument
+ freshenv: bool = False, # argument is not in the same order as in the superclass
+ confoverrides: dict[str, Any] | None = None,
+ status: StringIO | None = None,
+ warning: StringIO | None = None,
+ tags: Sequence[str] = (),
+ docutils_conf: str | None = None, # extra constructor argument
+ parallel: int = 0,
+ # additional arguments at the end to keep the signature
+ verbosity: int = 0, # argument is not in the same order as in the superclass
+ warningiserror: bool = False, # argument is not in the same order as in the superclass
+ pdb: bool = False,
+ exception_on_warning: bool = False,
+ # unknown keyword arguments
+ **extras: Any,
+ ) -> None:
assert srcdir is not None
+
if verbosity == -1:
quiet = True
verbosity = 0
else:
quiet = False
+
if status is None:
+ # ensure that :attr:`status` is a StringIO and not sys.stdout
+ # but allow the stream to be /dev/null by passing verbosity=-1
status = None if quiet else StringIO()
elif not isinstance(status, StringIO):
- err = '%r must be an io.StringIO object, got: %s' % ('status',
- type(status))
+ err = "%r must be an io.StringIO object, got: %s" % ('status', type(status))
raise TypeError(err)
+
if warning is None:
+ # ensure that :attr:`warning` is a StringIO and not sys.stderr
+ # but allow the stream to be /dev/null by passing verbosity=-1
warning = None if quiet else StringIO()
elif not isinstance(warning, StringIO):
- err = '%r must be an io.StringIO object, got: %s' % ('warning',
- type(warning))
+ err = '%r must be an io.StringIO object, got: %s' % ('warning', type(warning))
raise TypeError(err)
+
self.docutils_conf_path = srcdir / 'docutils.conf'
if docutils_conf is not None:
self.docutils_conf_path.write_text(docutils_conf, encoding='utf8')
+
if builddir is None:
builddir = srcdir / '_build'
+
confdir = srcdir
outdir = builddir.joinpath(buildername)
outdir.mkdir(parents=True, exist_ok=True)
@@ -86,32 +161,60 @@ class SphinxTestApp(sphinx.application.Sphinx):
doctreedir.mkdir(parents=True, exist_ok=True)
if confoverrides is None:
confoverrides = {}
+
self._saved_path = sys.path.copy()
self.extras: Mapping[str, Any] = MappingProxyType(extras)
"""Extras keyword arguments."""
+
try:
- super().__init__(srcdir, confdir, outdir, doctreedir,
- buildername, confoverrides=confoverrides, status=status,
- warning=warning, freshenv=freshenv, warningiserror=
- warningiserror, tags=tags, verbosity=verbosity, parallel=
- parallel, pdb=pdb, exception_on_warning=exception_on_warning)
+ super().__init__(
+ srcdir, confdir, outdir, doctreedir, buildername,
+ confoverrides=confoverrides, status=status, warning=warning,
+ freshenv=freshenv, warningiserror=warningiserror, tags=tags,
+ verbosity=verbosity, parallel=parallel,
+ pdb=pdb, exception_on_warning=exception_on_warning,
+ )
except Exception:
self.cleanup()
raise
+ def _init_builder(self) -> None:
+ # override the default theme to 'basic' rather than 'alabaster'
+ # for test independence
+
+ if 'html_theme' in self.config._overrides:
+ pass # respect overrides
+ elif 'html_theme' in self.config and self.config.html_theme == 'alabaster':
+ self.config.html_theme = self.config._overrides.get('html_theme', 'basic')
+ super()._init_builder()
+
@property
- def status(self) ->StringIO:
+ def status(self) -> StringIO:
"""The in-memory text I/O for the application status messages."""
- pass
+ # sphinx.application.Sphinx uses StringIO for a quiet stream
+ assert isinstance(self._status, StringIO)
+ return self._status
@property
- def warning(self) ->StringIO:
+ def warning(self) -> StringIO:
"""The in-memory text I/O for the application warning messages."""
- pass
+ # sphinx.application.Sphinx uses StringIO for a quiet stream
+ assert isinstance(self._warning, StringIO)
+ return self._warning
+
+ def cleanup(self, doctrees: bool = False) -> None:
+ sys.path[:] = self._saved_path
+ _clean_up_global_state()
+ with contextlib.suppress(FileNotFoundError):
+ os.remove(self.docutils_conf_path)
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return f'<{self.__class__.__name__} buildername={self.builder.name!r}>'
+ def build(self, force_all: bool = False, filenames: list[str] | None = None) -> None:
+ self.env._pickled_doctree_cache.clear()
+ super().build(force_all, filenames)
+
class SphinxTestAppWrapperForSkipBuilding(SphinxTestApp):
"""A wrapper for SphinxTestApp.
@@ -120,16 +223,44 @@ class SphinxTestAppWrapperForSkipBuilding(SphinxTestApp):
if it has already been built and there are any output files.
"""
+ def build(self, force_all: bool = False, filenames: list[str] | None = None) -> None:
+ if not os.listdir(self.outdir):
+ # if listdir is empty, do build.
+ super().build(force_all, filenames)
+ # otherwise, we can use built cache
+
+
+def _clean_up_global_state() -> None:
+ # clean up Docutils global state
+ directives._directives.clear() # type: ignore[attr-defined]
+ roles._roles.clear() # type: ignore[attr-defined]
+ for node in additional_nodes:
+ delattr(nodes.GenericNodeVisitor, f'visit_{node.__name__}')
+ delattr(nodes.GenericNodeVisitor, f'depart_{node.__name__}')
+ delattr(nodes.SparseNodeVisitor, f'visit_{node.__name__}')
+ delattr(nodes.SparseNodeVisitor, f'depart_{node.__name__}')
+ additional_nodes.clear()
+
+ # clean up Sphinx global state
+ sphinx.locale.translators.clear()
+ # clean up autodoc global state
+ sphinx.pycode.ModuleAnalyzer.cache.clear()
+
+
+# deprecated name -> (object to return, canonical path or '', removal version)
_DEPRECATED_OBJECTS: dict[str, tuple[Any, str, tuple[int, int]]] = {
- 'strip_escseq': (strip_colors, 'sphinx.util.console.strip_colors', (9, 0))}
+ 'strip_escseq': (strip_colors, 'sphinx.util.console.strip_colors', (9, 0)),
+}
-def __getattr__(name: str) ->Any:
+def __getattr__(name: str) -> Any:
if name not in _DEPRECATED_OBJECTS:
msg = f'module {__name__!r} has no attribute {name!r}'
raise AttributeError(msg)
+
from sphinx.deprecation import _deprecation_warning
+
deprecated_object, canonical_name, remove = _DEPRECATED_OBJECTS[name]
_deprecation_warning(__name__, name, canonical_name, remove=remove)
return deprecated_object
diff --git a/sphinx/theming.py b/sphinx/theming.py
index 43bb6bd9b..6d9986ba3 100644
--- a/sphinx/theming.py
+++ b/sphinx/theming.py
@@ -1,6 +1,9 @@
"""Theming support for HTML builders."""
+
from __future__ import annotations
-__all__ = 'Theme', 'HTMLThemeFactory'
+
+__all__ = ('Theme', 'HTMLThemeFactory')
+
import configparser
import contextlib
import os
@@ -11,39 +14,45 @@ from importlib.metadata import entry_points
from os import path
from typing import TYPE_CHECKING, Any
from zipfile import ZipFile
+
from sphinx import package_dir
from sphinx.config import check_confval_types as _config_post_init
from sphinx.errors import ThemeError
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.osutil import ensuredir
+
if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib
+
+
if TYPE_CHECKING:
from collections.abc import Callable
from typing import TypedDict
+
from typing_extensions import Required
- from sphinx.application import Sphinx
+ from sphinx.application import Sphinx
- class _ThemeToml(TypedDict, total=(False)):
+ class _ThemeToml(TypedDict, total=False):
theme: Required[_ThemeTomlTheme]
options: dict[str, str]
-
- class _ThemeTomlTheme(TypedDict, total=(False)):
+ class _ThemeTomlTheme(TypedDict, total=False):
inherit: Required[str]
stylesheets: list[str]
sidebars: list[str]
pygments_style: _ThemeTomlThemePygments
-
- class _ThemeTomlThemePygments(TypedDict, total=(False)):
+ class _ThemeTomlThemePygments(TypedDict, total=False):
default: str
dark: str
+
+
logger = logging.getLogger(__name__)
+
_NO_DEFAULT = object()
_THEME_TOML = 'theme.toml'
_THEME_CONF = 'theme.conf'
@@ -55,11 +64,18 @@ class Theme:
This class supports both theme directory and theme archive (zipped theme).
"""
- def __init__(self, name: str, *, configs: dict[str, _ConfigFile], paths:
- list[str], tmp_dirs: list[str]) ->None:
+ def __init__(
+ self,
+ name: str,
+ *,
+ configs: dict[str, _ConfigFile],
+ paths: list[str],
+ tmp_dirs: list[str],
+ ) -> None:
self.name = name
self._dirs = tuple(paths)
self._tmp_dirs = tmp_dirs
+
options: dict[str, Any] = {}
self.stylesheets: tuple[str, ...] = ()
self.sidebar_templates: tuple[str, ...] = ()
@@ -75,35 +91,71 @@ class Theme:
self.pygments_style_default = config.pygments_style_default
if config.pygments_style_dark is not None:
self.pygments_style_dark = config.pygments_style_dark
+
self._options = options
- def get_theme_dirs(self) ->list[str]:
+ def get_theme_dirs(self) -> list[str]:
"""Return a list of theme directories, beginning with this theme's,
then the base theme's, then that one's base theme's, etc.
"""
- pass
+ return list(self._dirs)
- def get_config(self, section: str, name: str, default: Any=_NO_DEFAULT
- ) ->Any:
+ def get_config(self, section: str, name: str, default: Any = _NO_DEFAULT) -> Any:
"""Return the value for a theme configuration setting, searching the
base theme chain.
"""
- pass
+ if section == 'theme':
+ if name == 'stylesheet':
+ value = ', '.join(self.stylesheets) or default
+ elif name == 'sidebars':
+ value = ', '.join(self.sidebar_templates) or default
+ elif name == 'pygments_style':
+ value = self.pygments_style_default or default
+ elif name == 'pygments_dark_style':
+ value = self.pygments_style_dark or default
+ else:
+ value = default
+ elif section == 'options':
+ value = self._options.get(name, default)
+ else:
+ msg = __(
+ 'Theme configuration sections other than [theme] and [options] '
+ 'are not supported (tried to get a value from %r).'
+ )
+ raise ThemeError(msg)
+ if value is _NO_DEFAULT:
+ msg = __('setting %s.%s occurs in none of the searched theme configs') % (
+ section,
+ name,
+ )
+ raise ThemeError(msg)
+ return value
- def get_options(self, overrides: (dict[str, Any] | None)=None) ->dict[
- str, Any]:
+ def get_options(self, overrides: dict[str, Any] | None = None) -> dict[str, Any]:
"""Return a dictionary of theme options and their values."""
- pass
+ if overrides is None:
+ overrides = {}
+
+ options = self._options.copy()
+ for option, value in overrides.items():
+ if option not in options:
+ logger.warning(__('unsupported theme option %r given'), option)
+ else:
+ options[option] = value
- def _cleanup(self) ->None:
+ return options
+
+ def _cleanup(self) -> None:
"""Remove temporary directories."""
- pass
+ for tmp_dir in self._tmp_dirs:
+ with contextlib.suppress(Exception):
+ shutil.rmtree(tmp_dir)
class HTMLThemeFactory:
"""A factory class for HTML Themes."""
- def __init__(self, app: Sphinx) ->None:
+ def __init__(self, app: Sphinx) -> None:
self._app = app
self._themes = app.registry.html_themes
self._entry_point_themes: dict[str, Callable[[], None]] = {}
@@ -112,73 +164,397 @@ class HTMLThemeFactory:
self._load_additional_themes(app.config.html_theme_path)
self._load_entry_point_themes()
- def _load_builtin_themes(self) ->None:
+ def _load_builtin_themes(self) -> None:
"""Load built-in themes."""
- pass
+ themes = self._find_themes(path.join(package_dir, 'themes'))
+ for name, theme in themes.items():
+ self._themes[name] = theme
- def _load_additional_themes(self, theme_paths: list[str]) ->None:
+ def _load_additional_themes(self, theme_paths: list[str]) -> None:
"""Load additional themes placed at specified directories."""
- pass
+ for theme_path in theme_paths:
+ abs_theme_path = path.abspath(path.join(self._app.confdir, theme_path))
+ themes = self._find_themes(abs_theme_path)
+ for name, theme in themes.items():
+ self._themes[name] = theme
- def _load_entry_point_themes(self) ->None:
+ def _load_entry_point_themes(self) -> None:
"""Try to load a theme with the specified name.
This uses the ``sphinx.html_themes`` entry point from package metadata.
"""
- pass
+ for entry_point in entry_points(group='sphinx.html_themes'):
+ if entry_point.name in self._themes:
+ continue # don't overwrite loaded themes
+
+ def _load_theme_closure(
+ # bind variables in the function definition
+ app: Sphinx = self._app,
+ theme_module: str = entry_point.module,
+ ) -> None:
+ app.setup_extension(theme_module)
+ _config_post_init(app, app.config)
+
+ self._entry_point_themes[entry_point.name] = _load_theme_closure
@staticmethod
- def _find_themes(theme_path: str) ->dict[str, str]:
+ def _find_themes(theme_path: str) -> dict[str, str]:
"""Search themes from specified directory."""
- pass
-
- def create(self, name: str) ->Theme:
+ themes: dict[str, str] = {}
+ if not path.isdir(theme_path):
+ return themes
+
+ for entry in os.listdir(theme_path):
+ pathname = path.join(theme_path, entry)
+ if path.isfile(pathname) and entry.lower().endswith('.zip'):
+ if _is_archived_theme(pathname):
+ name = entry[:-4]
+ themes[name] = pathname
+ else:
+ logger.warning(
+ __(
+ 'file %r on theme path is not a valid '
+ 'zipfile or contains no theme'
+ ),
+ entry,
+ )
+ else:
+ toml_path = path.join(pathname, _THEME_TOML)
+ conf_path = path.join(pathname, _THEME_CONF)
+ if path.isfile(toml_path) or path.isfile(conf_path):
+ themes[entry] = pathname
+
+ return themes
+
+ def create(self, name: str) -> Theme:
"""Create an instance of theme."""
- pass
-
-
-def _is_archived_theme(filename: str, /) ->bool:
+ if name in self._entry_point_themes:
+ # Load a deferred theme from an entry point
+ entry_point_loader = self._entry_point_themes[name]
+ entry_point_loader()
+ if name not in self._themes:
+ raise ThemeError(__('no theme named %r found (missing theme.toml?)') % name)
+
+ themes, theme_dirs, tmp_dirs = _load_theme_with_ancestors(
+ name,
+ self._themes,
+ self._entry_point_themes,
+ )
+ return Theme(name, configs=themes, paths=theme_dirs, tmp_dirs=tmp_dirs)
+
+
+def _is_archived_theme(filename: str, /) -> bool:
"""Check whether the specified file is an archived theme file or not."""
- pass
-
-
-def _extract_zip(filename: str, target_dir: str, /) ->None:
+ try:
+ with ZipFile(filename) as f:
+ namelist = frozenset(f.namelist())
+ return _THEME_TOML in namelist or _THEME_CONF in namelist
+ except Exception:
+ return False
+
+
+def _load_theme_with_ancestors(
+ name: str,
+ theme_paths: dict[str, str],
+ entry_point_themes: dict[str, Callable[[], None]],
+ /,
+) -> tuple[dict[str, _ConfigFile], list[str], list[str]]:
+ themes: dict[str, _ConfigFile] = {}
+ theme_dirs: list[str] = []
+ tmp_dirs: list[str] = []
+
+ # having 10+ theme ancestors is ludicrous
+ for _ in range(10):
+ inherit, theme_dir, tmp_dir, config = _load_theme(name, theme_paths[name])
+ theme_dirs.append(theme_dir)
+ if tmp_dir is not None:
+ tmp_dirs.append(tmp_dir)
+ themes[name] = config
+ if inherit == 'none':
+ break
+ if inherit in themes:
+ msg = __('The %r theme has circular inheritance') % name
+ raise ThemeError(msg)
+ if inherit in entry_point_themes and inherit not in theme_paths:
+ # Load a deferred theme from an entry point
+ entry_point_loader = entry_point_themes[inherit]
+ entry_point_loader()
+ if inherit not in theme_paths:
+ msg = __(
+ 'The %r theme inherits from %r, which is not a loaded theme. '
+ 'Loaded themes are: %s'
+ ) % (name, inherit, ', '.join(sorted(theme_paths)))
+ raise ThemeError(msg)
+ name = inherit
+ else:
+ msg = __('The %r theme has too many ancestors') % name
+ raise ThemeError(msg)
+
+ return themes, theme_dirs, tmp_dirs
+
+
+def _load_theme(
+ name: str, theme_path: str, /
+) -> tuple[str, str, str | None, _ConfigFile]:
+ if path.isdir(theme_path):
+ # already a directory, do nothing
+ tmp_dir = None
+ theme_dir = theme_path
+ else:
+ # extract the theme to a temp directory
+ tmp_dir = tempfile.mkdtemp('sxt')
+ theme_dir = path.join(tmp_dir, name)
+ _extract_zip(theme_path, theme_dir)
+
+ if path.isfile(toml_path := path.join(theme_dir, _THEME_TOML)):
+ _cfg_table = _load_theme_toml(toml_path)
+ inherit = _validate_theme_toml(_cfg_table, name)
+ config = _convert_theme_toml(_cfg_table)
+ elif path.isfile(conf_path := path.join(theme_dir, _THEME_CONF)):
+ _cfg_parser = _load_theme_conf(conf_path)
+ inherit = _validate_theme_conf(_cfg_parser, name)
+ config = _convert_theme_conf(_cfg_parser)
+ else:
+ raise ThemeError(__('no theme configuration file found in %r') % theme_dir)
+
+ return inherit, theme_dir, tmp_dir, config
+
+
+def _extract_zip(filename: str, target_dir: str, /) -> None:
"""Extract zip file to target directory."""
- pass
+ ensuredir(target_dir)
+
+ with ZipFile(filename) as archive:
+ for name in archive.namelist():
+ if name.endswith('/'):
+ continue
+ entry = path.join(target_dir, name)
+ ensuredir(path.dirname(entry))
+ with open(path.join(entry), 'wb') as fp:
+ fp.write(archive.read(name))
+
+
+def _load_theme_toml(config_file_path: str, /) -> _ThemeToml:
+ with open(config_file_path, encoding='utf-8') as f:
+ config_text = f.read()
+ c = tomllib.loads(config_text)
+ return {s: c[s] for s in ('theme', 'options') if s in c} # type: ignore[return-value]
+
+
+def _validate_theme_toml(cfg: _ThemeToml, name: str) -> str:
+ if 'theme' not in cfg:
+ msg = __('theme %r doesn\'t have the "theme" table') % name
+ raise ThemeError(msg)
+ theme = cfg['theme']
+ if not isinstance(theme, dict):
+ msg = __('The %r theme "[theme]" table is not a table') % name
+ raise ThemeError(msg)
+ inherit = theme.get('inherit', '')
+ if not inherit:
+ msg = __('The %r theme must define the "theme.inherit" setting') % name
+ raise ThemeError(msg)
+ if 'options' in cfg:
+ if not isinstance(cfg['options'], dict):
+ msg = __('The %r theme "[options]" table is not a table') % name
+ raise ThemeError(msg)
+ return inherit
+
+
+def _convert_theme_toml(cfg: _ThemeToml, /) -> _ConfigFile:
+ theme = cfg['theme']
+ if 'stylesheets' in theme:
+ stylesheets: tuple[str, ...] | None = tuple(theme['stylesheets'])
+ else:
+ stylesheets = None
+ if 'sidebars' in theme:
+ sidebar_templates: tuple[str, ...] | None = tuple(theme['sidebars'])
+ else:
+ sidebar_templates = None
+ pygments_table = theme.get('pygments_style', {})
+ if isinstance(pygments_table, str):
+ hint = f'pygments_style = {{ default = "{pygments_table}" }}'
+ msg = (
+ __('The "theme.pygments_style" setting must be a table. Hint: "%s"') % hint
+ )
+ raise ThemeError(msg)
+ pygments_style_default: str | None = pygments_table.get('default')
+ pygments_style_dark: str | None = pygments_table.get('dark')
+ return _ConfigFile(
+ stylesheets=stylesheets,
+ sidebar_templates=sidebar_templates,
+ pygments_style_default=pygments_style_default,
+ pygments_style_dark=pygments_style_dark,
+ options=cfg.get('options', {}),
+ )
+
+
+def _load_theme_conf(config_file_path: str, /) -> configparser.RawConfigParser:
+ c = configparser.RawConfigParser()
+ c.read(config_file_path, encoding='utf-8')
+ return c
+
+
+def _validate_theme_conf(cfg: configparser.RawConfigParser, name: str) -> str:
+ if not cfg.has_section('theme'):
+ raise ThemeError(__('theme %r doesn\'t have the "theme" table') % name)
+ if inherit := cfg.get('theme', 'inherit', fallback=None):
+ return inherit
+ msg = __('The %r theme must define the "theme.inherit" setting') % name
+ raise ThemeError(msg)
+
+
+def _convert_theme_conf(cfg: configparser.RawConfigParser, /) -> _ConfigFile:
+ if stylesheet := cfg.get('theme', 'stylesheet', fallback=''):
+ stylesheets: tuple[str, ...] | None = tuple(
+ map(str.strip, stylesheet.split(','))
+ )
+ else:
+ stylesheets = None
+ if sidebar := cfg.get('theme', 'sidebars', fallback=''):
+ sidebar_templates: tuple[str, ...] | None = tuple(
+ map(str.strip, sidebar.split(','))
+ )
+ else:
+ sidebar_templates = None
+ pygments_style_default: str | None = cfg.get(
+ 'theme', 'pygments_style', fallback=None
+ )
+ pygments_style_dark: str | None = cfg.get(
+ 'theme', 'pygments_dark_style', fallback=None
+ )
+ options = dict(cfg.items('options')) if cfg.has_section('options') else {}
+ return _ConfigFile(
+ stylesheets=stylesheets,
+ sidebar_templates=sidebar_templates,
+ pygments_style_default=pygments_style_default,
+ pygments_style_dark=pygments_style_dark,
+ options=options,
+ )
class _ConfigFile:
- __slots__ = ('stylesheets', 'sidebar_templates',
- 'pygments_style_default', 'pygments_style_dark', 'options')
-
- def __init__(self, stylesheets: (tuple[str, ...] | None),
- sidebar_templates: (tuple[str, ...] | None), pygments_style_default:
- (str | None), pygments_style_dark: (str | None), options: dict[str,
- str]) ->None:
+ __slots__ = (
+ 'stylesheets',
+ 'sidebar_templates',
+ 'pygments_style_default',
+ 'pygments_style_dark',
+ 'options',
+ )
+
+ def __init__(
+ self,
+ stylesheets: tuple[str, ...] | None,
+ sidebar_templates: tuple[str, ...] | None,
+ pygments_style_default: str | None,
+ pygments_style_dark: str | None,
+ options: dict[str, str],
+ ) -> None:
self.stylesheets: tuple[str, ...] | None = stylesheets
self.sidebar_templates: tuple[str, ...] | None = sidebar_templates
self.pygments_style_default: str | None = pygments_style_default
self.pygments_style_dark: str | None = pygments_style_dark
self.options: dict[str, str] = options.copy()
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return (
- f'{self.__class__.__qualname__}(stylesheets={self.stylesheets!r}, sidebar_templates={self.sidebar_templates!r}, pygments_style_default={self.pygments_style_default!r}, pygments_style_dark={self.pygments_style_dark!r}, options={self.options!r})'
- )
-
- def __eq__(self, other: object) ->bool:
+ f'{self.__class__.__qualname__}('
+ f'stylesheets={self.stylesheets!r}, '
+ f'sidebar_templates={self.sidebar_templates!r}, '
+ f'pygments_style_default={self.pygments_style_default!r}, '
+ f'pygments_style_dark={self.pygments_style_dark!r}, '
+ f'options={self.options!r})'
+ )
+
+ def __eq__(self, other: object) -> bool:
if isinstance(other, _ConfigFile):
- return (self.stylesheets == other.stylesheets and self.
- sidebar_templates == other.sidebar_templates and self.
- pygments_style_default == other.pygments_style_default and
- self.pygments_style_dark == other.pygments_style_dark and
- self.options == other.options)
+ return (
+ self.stylesheets == other.stylesheets
+ and self.sidebar_templates == other.sidebar_templates
+ and self.pygments_style_default == other.pygments_style_default
+ and self.pygments_style_dark == other.pygments_style_dark
+ and self.options == other.options
+ )
return NotImplemented
- def __hash__(self) ->int:
- return hash((self.__class__.__qualname__, self.stylesheets, self.
- sidebar_templates, self.pygments_style_default, self.
- pygments_style_dark, self.options))
+ def __hash__(self) -> int:
+ return hash((
+ self.__class__.__qualname__,
+ self.stylesheets,
+ self.sidebar_templates,
+ self.pygments_style_default,
+ self.pygments_style_dark,
+ self.options,
+ ))
+
+
+def _migrate_conf_to_toml(argv: list[str]) -> int:
+ if argv[:1] != ['conf_to_toml']:
+ raise SystemExit(0)
+ argv = argv[1:]
+ if len(argv) != 1:
+ print('Usage: python -m sphinx.theming conf_to_toml <theme path>') # NoQA: T201
+ raise SystemExit(1)
+ theme_dir = path.realpath(argv[0])
+ conf_path = path.join(theme_dir, _THEME_CONF)
+ if not path.isdir(theme_dir) or not path.isfile(conf_path):
+ print( # NoQA: T201
+ f'{theme_dir!r} must be a path to a theme directory containing a "theme.conf" file'
+ )
+ return 1
+ _cfg_parser = _load_theme_conf(conf_path)
+ if not _cfg_parser.has_section('theme'):
+ print('The "theme" table is missing.') # NoQA: T201
+ return 1
+ inherit = _cfg_parser.get('theme', 'inherit', fallback=None)
+ if not inherit:
+ print('The "theme.inherit" setting is missing.') # NoQA: T201
+ return 1
+
+ toml_lines = [
+ '[theme]',
+ f'inherit = "{inherit}"',
+ ]
+
+ stylesheet = _cfg_parser.get('theme', 'stylesheet', fallback=...)
+ if stylesheet == '':
+ toml_lines.append('stylesheets = []')
+ elif stylesheet is not ...:
+ toml_lines.append('stylesheets = [')
+ toml_lines.extend(f' "{s}",' for s in map(str.strip, stylesheet.split(',')))
+ toml_lines.append(']')
+
+ sidebar = _cfg_parser.get('theme', 'sidebars', fallback=...)
+ if sidebar == '':
+ toml_lines.append('sidebars = []')
+ elif sidebar is not ...:
+ toml_lines.append('sidebars = [')
+ toml_lines += [f' "{s}",' for s in map(str.strip, sidebar.split(','))]
+ toml_lines.append(']')
+
+ styles = []
+ default = _cfg_parser.get('theme', 'pygments_style', fallback=...)
+ if default is not ...:
+ styles.append(f'default = "{default}"')
+ dark = _cfg_parser.get('theme', 'pygments_dark_style', fallback=...)
+ if dark is not ...:
+ styles.append(f'dark = "{dark}"')
+ if styles:
+ toml_lines.append('pygments_style = { ' + ', '.join(styles) + ' }')
+
+ if _cfg_parser.has_section('options'):
+ toml_lines.append('')
+ toml_lines.append('[options]')
+ toml_lines += [
+ f'{key} = "{d}"'
+ for key, default in _cfg_parser.items('options')
+ if (d := default.replace('"', r'\"')) or True
+ ]
+
+ toml_path = path.join(theme_dir, _THEME_TOML)
+ with open(toml_path, 'w', encoding='utf-8') as f:
+ f.write('\n'.join(toml_lines) + '\n')
+ print(f'Written converted settings to {toml_path!r}') # NoQA: T201
+ return 0
if __name__ == '__main__':
diff --git a/sphinx/transforms/compact_bullet_list.py b/sphinx/transforms/compact_bullet_list.py
index 54ebc5311..acd863478 100644
--- a/sphinx/transforms/compact_bullet_list.py
+++ b/sphinx/transforms/compact_bullet_list.py
@@ -1,11 +1,17 @@
"""Docutils transforms used by Sphinx when reading documents."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes
+
from sphinx import addnodes
from sphinx.transforms import SphinxTransform
+
if TYPE_CHECKING:
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
@@ -17,7 +23,27 @@ class RefOnlyListChecker(nodes.GenericNodeVisitor):
single reference in it.
"""
- def invisible_visit(self, node: Node) ->None:
+ def default_visit(self, node: Node) -> None:
+ raise nodes.NodeFound
+
+ def visit_bullet_list(self, node: nodes.bullet_list) -> None:
+ pass
+
+ def visit_list_item(self, node: nodes.list_item) -> None:
+ children: list[Node] = [child for child in node.children
+ if not isinstance(child, nodes.Invisible)]
+ if len(children) != 1:
+ raise nodes.NodeFound
+ if not isinstance(children[0], nodes.paragraph):
+ raise nodes.NodeFound
+ para = children[0]
+ if len(para) != 1:
+ raise nodes.NodeFound
+ if not isinstance(para[0], addnodes.pending_xref):
+ raise nodes.NodeFound
+ raise nodes.SkipChildren
+
+ def invisible_visit(self, node: Node) -> None:
"""Invisible nodes should be ignored."""
pass
@@ -28,4 +54,38 @@ class RefOnlyBulletListTransform(SphinxTransform):
Specifically implemented for 'Indices and Tables' section, which looks
odd when html_compact_lists is false.
"""
+
default_priority = 100
+
+ def apply(self, **kwargs: Any) -> None:
+ if self.config.html_compact_lists:
+ return
+
+ def check_refonly_list(node: Node) -> bool:
+ """Check for list with only references in it."""
+ visitor = RefOnlyListChecker(self.document)
+ try:
+ node.walk(visitor)
+ except nodes.NodeFound:
+ return False
+ else:
+ return True
+
+ for node in self.document.findall(nodes.bullet_list):
+ if check_refonly_list(node):
+ for item in node.findall(nodes.list_item):
+ para = cast(nodes.paragraph, item[0])
+ ref = cast(nodes.reference, para[0])
+ compact_para = addnodes.compact_paragraph()
+ compact_para += ref
+ item.replace(para, compact_para)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_transform(RefOnlyBulletListTransform)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py
index 7d4cc25e7..8b6783676 100644
--- a/sphinx/transforms/i18n.py
+++ b/sphinx/transforms/i18n.py
@@ -1,12 +1,16 @@
"""Docutils transforms used by Sphinx when reading documents."""
+
from __future__ import annotations
+
import contextlib
from os import path
from re import DOTALL, match
from textwrap import indent
from typing import TYPE_CHECKING, Any, TypeVar
+
from docutils import nodes
from docutils.io import StringInput
+
from sphinx import addnodes
from sphinx.domains.std import make_glossary_term, split_term_classifiers
from sphinx.errors import ConfigError
@@ -16,19 +20,36 @@ from sphinx.transforms import SphinxTransform
from sphinx.util import get_filetype, logging
from sphinx.util.i18n import docname_to_domain
from sphinx.util.index_entries import split_index_msg
-from sphinx.util.nodes import IMAGE_TYPE_NODES, LITERAL_TYPE_NODES, NodeMatcher, extract_messages, traverse_translatable_index
+from sphinx.util.nodes import (
+ IMAGE_TYPE_NODES,
+ LITERAL_TYPE_NODES,
+ NodeMatcher,
+ extract_messages,
+ traverse_translatable_index,
+)
+
if TYPE_CHECKING:
from collections.abc import Sequence
+
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.util.typing import ExtensionMetadata
+
+
logger = logging.getLogger(__name__)
-EXCLUDED_PENDING_XREF_ATTRIBUTES = 'refexplicit',
+
+# The attributes not copied to the translated node
+#
+# * refexplict: For allow to give (or not to give) an explicit title
+# to the pending_xref on translation
+EXCLUDED_PENDING_XREF_ATTRIBUTES = ('refexplicit',)
+
+
N = TypeVar('N', bound=nodes.Node)
-def publish_msgstr(app: Sphinx, source: str, source_path: str, source_line:
- int, config: Config, settings: Any) ->nodes.Element:
+def publish_msgstr(app: Sphinx, source: str, source_path: str, source_line: int,
+ config: Config, settings: Any) -> nodes.Element:
"""Publish msgstr (single line) into docutils document
:param sphinx.application.Sphinx app: sphinx application
@@ -40,55 +61,567 @@ def publish_msgstr(app: Sphinx, source: str, source_path: str, source_line:
:return: document
:rtype: docutils.nodes.document
"""
- pass
+ try:
+ # clear rst_prolog temporarily
+ rst_prolog = config.rst_prolog
+ config.rst_prolog = None
+
+ from sphinx.io import SphinxI18nReader
+ reader = SphinxI18nReader()
+ reader.setup(app)
+ filetype = get_filetype(config.source_suffix, source_path)
+ parser = app.registry.create_source_parser(app, filetype)
+ doc = reader.read(
+ source=StringInput(source=source,
+ source_path=f"{source_path}:{source_line}:<translated>"),
+ parser=parser,
+ settings=settings,
+ )
+ with contextlib.suppress(IndexError): # empty node
+ return doc[0]
+ return doc
+ finally:
+ config.rst_prolog = rst_prolog
+
+
+def parse_noqa(source: str) -> tuple[str, bool]:
+ m = match(r"(.*)(?<!\\)#\s*noqa\s*$", source, DOTALL)
+ if m:
+ return m.group(1), True
+ else:
+ return source, False
class PreserveTranslatableMessages(SphinxTransform):
"""
Preserve original translatable messages before translation
"""
- default_priority = 10
+
+ default_priority = 10 # this MUST be invoked before Locale transform
+
+ def apply(self, **kwargs: Any) -> None:
+ for node in self.document.findall(addnodes.translatable):
+ node.preserve_original_messages()
class _NodeUpdater:
"""Contains logic for updating one node with the translated content."""
- def __init__(self, node: nodes.Element, patch: nodes.Element, document:
- nodes.document, noqa: bool) ->None:
+ def __init__(
+ self, node: nodes.Element, patch: nodes.Element, document: nodes.document, noqa: bool,
+ ) -> None:
self.node: nodes.Element = node
self.patch: nodes.Element = patch
self.document: nodes.document = document
self.noqa: bool = noqa
def compare_references(self, old_refs: Sequence[nodes.Element],
- new_refs: Sequence[nodes.Element], warning_msg: str) ->None:
+ new_refs: Sequence[nodes.Element],
+ warning_msg: str) -> None:
"""Warn about mismatches between references in original and translated content."""
- pass
+ # FIXME: could use a smarter strategy than len(old_refs) == len(new_refs)
+ if not self.noqa and len(old_refs) != len(new_refs):
+ old_ref_rawsources = [ref.rawsource for ref in old_refs]
+ new_ref_rawsources = [ref.rawsource for ref in new_refs]
+ logger.warning(warning_msg.format(old_ref_rawsources, new_ref_rawsources),
+ location=self.node, type='i18n', subtype='inconsistent_references')
+
+ def update_title_mapping(self) -> bool:
+ processed = False # skip flag
+
+ # update title(section) target name-id mapping
+ if isinstance(self.node, nodes.title) and isinstance(self.node.parent, nodes.section):
+ section_node = self.node.parent
+ new_name = nodes.fully_normalize_name(self.patch.astext())
+ old_name = nodes.fully_normalize_name(self.node.astext())
+
+ if old_name != new_name:
+ # if name would be changed, replace node names and
+ # document nameids mapping with new name.
+ names: list[str] = section_node.setdefault('names', [])
+ names.append(new_name)
+ # Original section name (reference target name) should be kept to refer
+ # from other nodes which is still not translated or uses explicit target
+ # name like "`text to display <explicit target name_>`_"..
+ # So, `old_name` is still exist in `names`.
+
+ _id = self.document.nameids.get(old_name, None)
+ explicit = self.document.nametypes.get(old_name, None)
+
+ # * if explicit: _id is label. title node need another id.
+ # * if not explicit:
+ #
+ # * if _id is None:
+ #
+ # _id is None means:
+ #
+ # 1. _id was not provided yet.
+ #
+ # 2. _id was duplicated.
+ #
+ # old_name entry still exists in nameids and
+ # nametypes for another duplicated entry.
+ #
+ # * if _id is provided: below process
+ if _id:
+ if not explicit:
+ # _id was not duplicated.
+ # remove old_name entry from document ids database
+ # to reuse original _id.
+ self.document.nameids.pop(old_name, None)
+ self.document.nametypes.pop(old_name, None)
+ self.document.ids.pop(_id, None)
+
+ # re-entry with new named section node.
+ #
+ # Note: msgnode that is a second parameter of the
+ # `note_implicit_target` is not necessary here because
+ # section_node has been noted previously on rst parsing by
+ # `docutils.parsers.rst.states.RSTState.new_subsection()`
+ # and already has `system_message` if needed.
+ self.document.note_implicit_target(section_node)
+
+ # replace target's refname to new target name
+ matcher = NodeMatcher(nodes.target, refname=old_name)
+ for old_target in matcher.findall(self.document):
+ old_target['refname'] = new_name
+
+ processed = True
+
+ return processed
+
+ def update_autofootnote_references(self) -> None:
+ # auto-numbered foot note reference should use original 'ids'.
+ def list_replace_or_append(lst: list[N], old: N, new: N) -> None:
+ if old in lst:
+ lst[lst.index(old)] = new
+ else:
+ lst.append(new)
+
+ is_autofootnote_ref = NodeMatcher(nodes.footnote_reference, auto=Any)
+ old_foot_refs = list(is_autofootnote_ref.findall(self.node))
+ new_foot_refs = list(is_autofootnote_ref.findall(self.patch))
+ self.compare_references(old_foot_refs, new_foot_refs,
+ __('inconsistent footnote references in translated message.'
+ ' original: {0}, translated: {1}'))
+ old_foot_namerefs: dict[str, list[nodes.footnote_reference]] = {}
+ for r in old_foot_refs:
+ old_foot_namerefs.setdefault(r.get('refname'), []).append(r)
+ for newf in new_foot_refs:
+ refname = newf.get('refname')
+ refs = old_foot_namerefs.get(refname, [])
+ if not refs:
+ newf.parent.remove(newf)
+ continue
+
+ oldf = refs.pop(0)
+ newf['ids'] = oldf['ids']
+ for id in newf['ids']:
+ self.document.ids[id] = newf
+
+ if newf['auto'] == 1:
+ # autofootnote_refs
+ list_replace_or_append(self.document.autofootnote_refs, oldf, newf)
+ else:
+ # symbol_footnote_refs
+ list_replace_or_append(self.document.symbol_footnote_refs, oldf, newf)
+
+ if refname:
+ footnote_refs = self.document.footnote_refs.setdefault(refname, [])
+ list_replace_or_append(footnote_refs, oldf, newf)
+
+ refnames = self.document.refnames.setdefault(refname, [])
+ list_replace_or_append(refnames, oldf, newf)
+
+ def update_refnamed_references(self) -> None:
+ # reference should use new (translated) 'refname'.
+ # * reference target ".. _Python: ..." is not translatable.
+ # * use translated refname for section refname.
+ # * inline reference "`Python <...>`_" has no 'refname'.
+ is_refnamed_ref = NodeMatcher(nodes.reference, refname=Any)
+ old_refs = list(is_refnamed_ref.findall(self.node))
+ new_refs = list(is_refnamed_ref.findall(self.patch))
+ self.compare_references(old_refs, new_refs,
+ __('inconsistent references in translated message.'
+ ' original: {0}, translated: {1}'))
+ old_ref_names = [r['refname'] for r in old_refs]
+ new_ref_names = [r['refname'] for r in new_refs]
+ orphans = [*({*old_ref_names} - {*new_ref_names})]
+ for newr in new_refs:
+ if not self.document.has_name(newr['refname']):
+ # Maybe refname is translated but target is not translated.
+ # Note: multiple translated refnames break link ordering.
+ if orphans:
+ newr['refname'] = orphans.pop(0)
+ else:
+ # orphan refnames is already empty!
+ # reference number is same in new_refs and old_refs.
+ pass
+
+ self.document.note_refname(newr)
+
+ def update_refnamed_footnote_references(self) -> None:
+ # refnamed footnote should use original 'ids'.
+ is_refnamed_footnote_ref = NodeMatcher(nodes.footnote_reference, refname=Any)
+ old_foot_refs = list(is_refnamed_footnote_ref.findall(self.node))
+ new_foot_refs = list(is_refnamed_footnote_ref.findall(self.patch))
+ refname_ids_map: dict[str, list[str]] = {}
+ self.compare_references(old_foot_refs, new_foot_refs,
+ __('inconsistent footnote references in translated message.'
+ ' original: {0}, translated: {1}'))
+ for oldf in old_foot_refs:
+ refname_ids_map.setdefault(oldf["refname"], []).append(oldf["ids"])
+ for newf in new_foot_refs:
+ refname = newf["refname"]
+ if refname_ids_map.get(refname):
+ newf["ids"] = refname_ids_map[refname].pop(0)
+
+ def update_citation_references(self) -> None:
+ # citation should use original 'ids'.
+ is_citation_ref = NodeMatcher(nodes.citation_reference, refname=Any)
+ old_cite_refs = list(is_citation_ref.findall(self.node))
+ new_cite_refs = list(is_citation_ref.findall(self.patch))
+ self.compare_references(old_cite_refs, new_cite_refs,
+ __('inconsistent citation references in translated message.'
+ ' original: {0}, translated: {1}'))
+ refname_ids_map: dict[str, list[str]] = {}
+ for oldc in old_cite_refs:
+ refname_ids_map.setdefault(oldc["refname"], []).append(oldc["ids"])
+ for newc in new_cite_refs:
+ refname = newc["refname"]
+ if refname_ids_map.get(refname):
+ newc["ids"] = refname_ids_map[refname].pop()
+
+ def update_pending_xrefs(self) -> None:
+ # Original pending_xref['reftarget'] contain not-translated
+ # target name, new pending_xref must use original one.
+ # This code restricts to change ref-targets in the translation.
+ old_xrefs = [*self.node.findall(addnodes.pending_xref)]
+ new_xrefs = [*self.patch.findall(addnodes.pending_xref)]
+ self.compare_references(old_xrefs, new_xrefs,
+ __('inconsistent term references in translated message.'
+ ' original: {0}, translated: {1}'))
+
+ xref_reftarget_map: dict[tuple[str, str, str] | None, dict[str, Any]] = {}
+
+ def get_ref_key(node: addnodes.pending_xref) -> tuple[str, str, str] | None:
+ case = node["refdomain"], node["reftype"]
+ if case == ('std', 'term'):
+ return None
+ else:
+ return (
+ node["refdomain"],
+ node["reftype"],
+ node['reftarget'],
+ )
+
+ for old in old_xrefs:
+ key = get_ref_key(old)
+ if key:
+ xref_reftarget_map[key] = old.attributes
+ for new in new_xrefs:
+ key = get_ref_key(new)
+ # Copy attributes to keep original node behavior. Especially
+ # copying 'reftarget', 'py:module', 'py:class' are needed.
+ for k, v in xref_reftarget_map.get(key, {}).items():
+ if k not in EXCLUDED_PENDING_XREF_ATTRIBUTES:
+ new[k] = v
+
+ def update_leaves(self) -> None:
+ for child in self.patch.children:
+ child.parent = self.node
+ self.node.children = self.patch.children
class Locale(SphinxTransform):
"""
Replace translatable nodes with their translated doctree.
"""
+
default_priority = 20
+ def apply(self, **kwargs: Any) -> None:
+ settings, source = self.document.settings, self.document['source']
+ msgstr = ''
+
+ textdomain = docname_to_domain(self.env.docname, self.config.gettext_compact)
+
+ # fetch translations
+ dirs = [path.join(self.env.srcdir, directory)
+ for directory in self.config.locale_dirs]
+ catalog, has_catalog = init_locale(dirs, self.config.language, textdomain)
+ if not has_catalog:
+ return
+
+ catalogues = [getattr(catalog, '_catalog', None)]
+ while (catalog := catalog._fallback) is not None: # type: ignore[attr-defined]
+ catalogues.append(getattr(catalog, '_catalog', None))
+ merged: dict[str, str] = {}
+ for catalogue in filter(None, reversed(catalogues)): # type: dict[str, str]
+ merged |= catalogue
+
+ # phase1: replace reference ids with translated names
+ for node, msg in extract_messages(self.document):
+ msgstr = merged.get(msg, '')
+
+ # There is no point in having noqa on literal blocks because
+ # they cannot contain references. Recognizing it would just
+ # completely prevent escaping the noqa. Outside of literal
+ # blocks, one can always write \#noqa.
+ if not isinstance(node, LITERAL_TYPE_NODES):
+ msgstr, _ = parse_noqa(msgstr)
+
+ if msgstr.strip() == '':
+ # as-of-yet untranslated
+ node['translated'] = False
+ continue
+ if msgstr == msg:
+ # identical source and translated messages
+ node['translated'] = True
+ continue
+
+ # Avoid "Literal block expected; none found." warnings.
+ # If msgstr ends with '::' then it cause warning message at
+ # parser.parse() processing.
+ # literal-block-warning is only appear in avobe case.
+ if msgstr.strip().endswith('::'):
+ msgstr += '\n\n dummy literal'
+ # dummy literal node will discard by 'patch = patch[0]'
+
+ # literalblock need literal block notation to avoid it become
+ # paragraph.
+ if isinstance(node, LITERAL_TYPE_NODES):
+ msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
+
+ patch = publish_msgstr(self.app, msgstr, source,
+ node.line, self.config, settings) # type: ignore[arg-type]
+ # FIXME: no warnings about inconsistent references in this part
+ # XXX doctest and other block markup
+ if not isinstance(patch, nodes.paragraph):
+ continue # skip for now
+
+ updater = _NodeUpdater(node, patch, self.document, noqa=False)
+ processed = updater.update_title_mapping()
+
+ # glossary terms update refid
+ if isinstance(node, nodes.term):
+ for _id in node['ids']:
+ term, first_classifier = split_term_classifiers(msgstr)
+ patch = publish_msgstr(
+ self.app, term or '', source, node.line, self.config, settings, # type: ignore[arg-type]
+ )
+ updater.patch = make_glossary_term(
+ self.env, patch, first_classifier,
+ source, node.line, _id, self.document, # type: ignore[arg-type]
+ )
+ processed = True
+
+ # update leaves with processed nodes
+ if processed:
+ updater.update_leaves()
+ node['translated'] = True # to avoid double translation
+ else:
+ node['translated'] = False
+
+ # phase2: translation
+ for node, msg in extract_messages(self.document):
+ if node.setdefault('translated', False): # to avoid double translation
+ continue # skip if the node is already translated by phase1
+
+ msgstr = merged.get(msg, '')
+ noqa = False
+
+ # See above.
+ if not isinstance(node, LITERAL_TYPE_NODES):
+ msgstr, noqa = parse_noqa(msgstr)
+
+ if not msgstr or msgstr == msg: # as-of-yet untranslated
+ node['translated'] = False
+ continue
+
+ # update translatable nodes
+ if isinstance(node, addnodes.translatable):
+ node.apply_translated_message(msg, msgstr)
+ continue
+
+ # update meta nodes
+ if isinstance(node, nodes.meta):
+ node['content'] = msgstr
+ node['translated'] = True
+ continue
+
+ if isinstance(node, nodes.image) and node.get('alt') == msg:
+ node['alt'] = msgstr
+ continue
+
+ # Avoid "Literal block expected; none found." warnings.
+ # If msgstr ends with '::' then it cause warning message at
+ # parser.parse() processing.
+ # literal-block-warning is only appear in avobe case.
+ if msgstr.strip().endswith('::'):
+ msgstr += '\n\n dummy literal'
+ # dummy literal node will discard by 'patch = patch[0]'
+
+ # literalblock need literal block notation to avoid it become
+ # paragraph.
+ if isinstance(node, LITERAL_TYPE_NODES):
+ msgstr = '::\n\n' + indent(msgstr, ' ' * 3)
+
+ # Structural Subelements phase1
+ # There is a possibility that only the title node is created.
+ # see: https://docutils.sourceforge.io/docs/ref/doctree.html#structural-subelements
+ if isinstance(node, nodes.title):
+ # This generates: <section ...><title>msgstr</title></section>
+ msgstr = msgstr + '\n' + '=' * len(msgstr) * 2
+
+ patch = publish_msgstr(self.app, msgstr, source,
+ node.line, self.config, settings) # type: ignore[arg-type]
+ # Structural Subelements phase2
+ if isinstance(node, nodes.title):
+ # get <title> node that placed as a first child
+ patch = patch.next_node() # type: ignore[assignment]
+
+ # ignore unexpected markups in translation message
+ unexpected: tuple[type[nodes.Element], ...] = (
+ nodes.paragraph, # expected form of translation
+ nodes.title, # generated by above "Subelements phase2"
+ )
+
+ # following types are expected if
+ # config.gettext_additional_targets is configured
+ unexpected += LITERAL_TYPE_NODES
+ unexpected += IMAGE_TYPE_NODES
+
+ if not isinstance(patch, unexpected):
+ continue # skip
+
+ updater = _NodeUpdater(node, patch, self.document, noqa)
+ updater.update_autofootnote_references()
+ updater.update_refnamed_references()
+ updater.update_refnamed_footnote_references()
+ updater.update_citation_references()
+ updater.update_pending_xrefs()
+ updater.update_leaves()
+
+ # for highlighting that expects .rawsource and .astext() are same.
+ if isinstance(node, LITERAL_TYPE_NODES):
+ node.rawsource = node.astext()
+
+ if isinstance(node, nodes.image) and node.get('alt') != msg:
+ node['uri'] = patch['uri']
+ node['translated'] = False
+ continue # do not mark translated
+
+ node['translated'] = True # to avoid double translation
+
+ if 'index' in self.config.gettext_additional_targets:
+ # Extract and translate messages for index entries.
+ for node, entries in traverse_translatable_index(self.document):
+ new_entries: list[tuple[str, str, str, str, str | None]] = []
+ for entry_type, value, target_id, main, _category_key in entries:
+ msg_parts = split_index_msg(entry_type, value)
+ msgstr_parts = []
+ for part in msg_parts:
+ msgstr = merged.get(part, '')
+ if not msgstr:
+ msgstr = part
+ msgstr_parts.append(msgstr)
+
+ new_entry = entry_type, ';'.join(msgstr_parts), target_id, main, None
+ new_entries.append(new_entry)
+
+ node['raw_entries'] = entries
+ node['entries'] = new_entries
+
class TranslationProgressTotaliser(SphinxTransform):
"""
Calculate the number of translated and untranslated nodes.
"""
- default_priority = 25
+
+ default_priority = 25 # MUST happen after Locale
+
+ def apply(self, **kwargs: Any) -> None:
+ from sphinx.builders.gettext import MessageCatalogBuilder
+ if isinstance(self.app.builder, MessageCatalogBuilder):
+ return
+
+ total = translated = 0
+ for node in NodeMatcher(nodes.Element, translated=Any).findall(self.document):
+ total += 1
+ if node['translated']:
+ translated += 1
+
+ self.document['translation_progress'] = {
+ 'total': total,
+ 'translated': translated,
+ }
class AddTranslationClasses(SphinxTransform):
"""
Add ``translated`` or ``untranslated`` classes to indicate translation status.
"""
+
default_priority = 950
+ def apply(self, **kwargs: Any) -> None:
+ from sphinx.builders.gettext import MessageCatalogBuilder
+ if isinstance(self.app.builder, MessageCatalogBuilder):
+ return
+
+ if not self.config.translation_progress_classes:
+ return
+
+ if self.config.translation_progress_classes is True:
+ add_translated = add_untranslated = True
+ elif self.config.translation_progress_classes == 'translated':
+ add_translated = True
+ add_untranslated = False
+ elif self.config.translation_progress_classes == 'untranslated':
+ add_translated = False
+ add_untranslated = True
+ else:
+ msg = ('translation_progress_classes must be '
+ 'True, False, "translated" or "untranslated"')
+ raise ConfigError(msg)
+
+ for node in NodeMatcher(nodes.Element, translated=Any).findall(self.document):
+ if node['translated']:
+ if add_translated:
+ node.setdefault('classes', []).append('translated') # type: ignore[arg-type]
+ else:
+ if add_untranslated:
+ node.setdefault('classes', []).append('untranslated') # type: ignore[arg-type]
+
class RemoveTranslatableInline(SphinxTransform):
"""
Remove inline nodes used for translation as placeholders.
"""
+
default_priority = 999
+
+ def apply(self, **kwargs: Any) -> None:
+ from sphinx.builders.gettext import MessageCatalogBuilder
+ if isinstance(self.app.builder, MessageCatalogBuilder):
+ return
+
+ matcher = NodeMatcher(nodes.inline, translatable=Any)
+ for inline in matcher.findall(self.document):
+ inline.parent.remove(inline)
+ inline.parent += inline.children
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_transform(PreserveTranslatableMessages)
+ app.add_transform(Locale)
+ app.add_transform(TranslationProgressTotaliser)
+ app.add_transform(AddTranslationClasses)
+ app.add_transform(RemoveTranslatableInline)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/transforms/post_transforms/code.py b/sphinx/transforms/post_transforms/code.py
index b9d48014f..4375b4d89 100644
--- a/sphinx/transforms/post_transforms/code.py
+++ b/sphinx/transforms/post_transforms/code.py
@@ -1,14 +1,20 @@
"""transforms for code-blocks."""
+
from __future__ import annotations
+
import sys
from typing import TYPE_CHECKING, Any, NamedTuple
+
from docutils import nodes
from pygments.lexers import PythonConsoleLexer, guess_lexer
+
from sphinx import addnodes
from sphinx.ext import doctest
from sphinx.transforms import SphinxTransform
+
if TYPE_CHECKING:
from docutils.nodes import Node, TextElement
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
@@ -27,17 +33,56 @@ class HighlightLanguageTransform(SphinxTransform):
:rst:dir:`highlight` directive. After processing, this transform
removes ``highlightlang`` node from doctree.
"""
+
default_priority = 400
+ def apply(self, **kwargs: Any) -> None:
+ visitor = HighlightLanguageVisitor(self.document,
+ self.config.highlight_language)
+ self.document.walkabout(visitor)
-class HighlightLanguageVisitor(nodes.NodeVisitor):
+ for node in list(self.document.findall(addnodes.highlightlang)):
+ node.parent.remove(node)
- def __init__(self, document: nodes.document, default_language: str) ->None:
- self.default_setting = HighlightSetting(default_language, False,
- sys.maxsize)
+
+class HighlightLanguageVisitor(nodes.NodeVisitor):
+ def __init__(self, document: nodes.document, default_language: str) -> None:
+ self.default_setting = HighlightSetting(default_language, False, sys.maxsize)
self.settings: list[HighlightSetting] = []
super().__init__(document)
+ def unknown_visit(self, node: Node) -> None:
+ pass
+
+ def unknown_departure(self, node: Node) -> None:
+ pass
+
+ def visit_document(self, node: Node) -> None:
+ self.settings.append(self.default_setting)
+
+ def depart_document(self, node: Node) -> None:
+ self.settings.pop()
+
+ def visit_start_of_file(self, node: Node) -> None:
+ self.settings.append(self.default_setting)
+
+ def depart_start_of_file(self, node: Node) -> None:
+ self.settings.pop()
+
+ def visit_highlightlang(self, node: addnodes.highlightlang) -> None:
+ self.settings[-1] = HighlightSetting(node['lang'],
+ node['force'],
+ node['linenothreshold'])
+
+ def visit_literal_block(self, node: nodes.literal_block) -> None:
+ setting = self.settings[-1]
+ if 'language' not in node:
+ node['language'] = setting.language
+ node['force'] = setting.force
+ if 'linenos' not in node:
+ lines = node.astext().count('\n')
+ node['linenos'] = (lines >= setting.lineno_threshold - 1)
+
class TrimDoctestFlagsTransform(SphinxTransform):
"""
@@ -45,4 +90,53 @@ class TrimDoctestFlagsTransform(SphinxTransform):
see :confval:`trim_doctest_flags` for more information.
"""
+
default_priority = HighlightLanguageTransform.default_priority + 1
+
+ def apply(self, **kwargs: Any) -> None:
+ for lbnode in self.document.findall(nodes.literal_block):
+ if self.is_pyconsole(lbnode):
+ self.strip_doctest_flags(lbnode)
+
+ for dbnode in self.document.findall(nodes.doctest_block):
+ self.strip_doctest_flags(dbnode)
+
+ def strip_doctest_flags(self, node: TextElement) -> None:
+ if not node.get('trim_flags', self.config.trim_doctest_flags):
+ return
+
+ source = node.rawsource
+ source = doctest.blankline_re.sub('', source)
+ source = doctest.doctestopt_re.sub('', source)
+ node.rawsource = source
+ node[:] = [nodes.Text(source)]
+
+ @staticmethod
+ def is_pyconsole(node: nodes.literal_block) -> bool:
+ if node.rawsource != node.astext():
+ return False # skip parsed-literal node
+
+ language = node.get('language')
+ if language in {'pycon', 'pycon3'}:
+ return True
+ elif language in {'py', 'python', 'py3', 'python3', 'default'}:
+ return node.rawsource.startswith('>>>')
+ elif language == 'guess':
+ try:
+ lexer = guess_lexer(node.rawsource)
+ return isinstance(lexer, PythonConsoleLexer)
+ except Exception:
+ pass
+
+ return False
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_post_transform(HighlightLanguageTransform)
+ app.add_post_transform(TrimDoctestFlagsTransform)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/transforms/post_transforms/images.py b/sphinx/transforms/post_transforms/images.py
index 76d5727d2..05b07dd22 100644
--- a/sphinx/transforms/post_transforms/images.py
+++ b/sphinx/transforms/post_transforms/images.py
@@ -1,12 +1,16 @@
"""Docutils transforms used by Sphinx."""
+
from __future__ import annotations
+
import os
import re
from hashlib import sha1
from math import ceil
from pathlib import Path
from typing import TYPE_CHECKING, Any
+
from docutils import nodes
+
from sphinx.locale import __
from sphinx.transforms import SphinxTransform
from sphinx.util import logging, requests
@@ -14,25 +18,146 @@ from sphinx.util._pathlib import _StrPath
from sphinx.util.http_date import epoch_to_rfc1123, rfc1123_to_epoch
from sphinx.util.images import get_image_extension, guess_mimetype, parse_data_uri
from sphinx.util.osutil import ensuredir
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
logger = logging.getLogger(__name__)
+
MAX_FILENAME_LEN = 32
CRITICAL_PATH_CHAR_RE = re.compile('[:;<>|*" ]')
class BaseImageConverter(SphinxTransform):
- pass
+ def apply(self, **kwargs: Any) -> None:
+ for node in self.document.findall(nodes.image):
+ if self.match(node):
+ self.handle(node)
+
+ def match(self, node: nodes.image) -> bool:
+ return True
+
+ def handle(self, node: nodes.image) -> None:
+ pass
+
+ @property
+ def imagedir(self) -> str:
+ return os.path.join(self.app.doctreedir, 'images')
class ImageDownloader(BaseImageConverter):
default_priority = 100
+ def match(self, node: nodes.image) -> bool:
+ if not self.app.builder.supported_image_types:
+ return False
+ if self.app.builder.supported_remote_images:
+ return False
+ return '://' in node['uri']
+
+ def handle(self, node: nodes.image) -> None:
+ try:
+ basename = os.path.basename(node['uri'])
+ if '?' in basename:
+ basename = basename.split('?')[0]
+ if basename == '' or len(basename) > MAX_FILENAME_LEN:
+ filename, ext = os.path.splitext(node['uri'])
+ basename = sha1(filename.encode(), usedforsecurity=False).hexdigest() + ext
+ basename = CRITICAL_PATH_CHAR_RE.sub("_", basename)
+
+ uri_hash = sha1(node['uri'].encode(), usedforsecurity=False).hexdigest()
+ path = Path(self.imagedir, uri_hash, basename)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ self._download_image(node, path)
+
+ except Exception as exc:
+ msg = __('Could not fetch remote image: %s [%s]')
+ logger.warning(msg, node['uri'], exc)
+
+ def _download_image(self, node: nodes.image, path: Path) -> None:
+ headers = {}
+ if path.exists():
+ timestamp: float = ceil(path.stat().st_mtime)
+ headers['If-Modified-Since'] = epoch_to_rfc1123(timestamp)
+
+ config = self.app.config
+ r = requests.get(
+ node['uri'], headers=headers,
+ _user_agent=config.user_agent,
+ _tls_info=(config.tls_verify, config.tls_cacerts),
+ )
+ if r.status_code >= 400:
+ msg = __('Could not fetch remote image: %s [%d]')
+ logger.warning(msg, node['uri'], r.status_code)
+ else:
+ self.app.env.original_image_uri[_StrPath(path)] = node['uri']
+
+ if r.status_code == 200:
+ path.write_bytes(r.content)
+ if last_modified := r.headers.get('Last-Modified'):
+ timestamp = rfc1123_to_epoch(last_modified)
+ os.utime(path, (timestamp, timestamp))
+
+ self._process_image(node, path)
+
+ def _process_image(self, node: nodes.image, path: Path) -> None:
+ str_path = _StrPath(path)
+ self.app.env.original_image_uri[str_path] = node['uri']
+
+ mimetype = guess_mimetype(path, default='*')
+ if mimetype != '*' and path.suffix == '':
+ # append a suffix if URI does not contain suffix
+ ext = get_image_extension(mimetype) or ''
+ with_ext = path.with_name(path.name + ext)
+ os.replace(path, with_ext)
+ self.app.env.original_image_uri.pop(str_path)
+ self.app.env.original_image_uri[_StrPath(with_ext)] = node['uri']
+ path = with_ext
+ path_str = str(path)
+ node['candidates'].pop('?')
+ node['candidates'][mimetype] = path_str
+ node['uri'] = path_str
+ self.app.env.images.add_file(self.env.docname, path_str)
+
class DataURIExtractor(BaseImageConverter):
default_priority = 150
+ def match(self, node: nodes.image) -> bool:
+ if self.app.builder.supported_data_uri_images is True:
+ return False # do not transform the image; data URIs are valid in the build output
+ return node['uri'].startswith('data:')
+
+ def handle(self, node: nodes.image) -> None:
+ image = parse_data_uri(node['uri'])
+ assert image is not None
+ ext = get_image_extension(image.mimetype)
+ if ext is None:
+ logger.warning(__('Unknown image format: %s...'), node['uri'][:32],
+ location=node)
+ return
+
+ ensuredir(os.path.join(self.imagedir, 'embeded'))
+ digest = sha1(image.data, usedforsecurity=False).hexdigest()
+ path = _StrPath(self.imagedir, 'embeded', digest + ext)
+ self.app.env.original_image_uri[path] = node['uri']
+
+ with open(path, 'wb') as f:
+ f.write(image.data)
+
+ path_str = str(path)
+ node['candidates'].pop('?')
+ node['candidates'][image.mimetype] = path_str
+ node['uri'] = path_str
+ self.app.env.images.add_file(self.env.docname, path_str)
+
+
+def get_filename_for(filename: str, mimetype: str) -> str:
+ basename = os.path.basename(filename)
+ basename = CRITICAL_PATH_CHAR_RE.sub("_", basename)
+ return os.path.splitext(basename)[0] + (get_image_extension(mimetype) or '')
+
class ImageConverter(BaseImageConverter):
"""A base class for image converters.
@@ -55,18 +180,113 @@ class ImageConverter(BaseImageConverter):
3. Register your image converter to Sphinx using
:py:meth:`.Sphinx.add_post_transform`
"""
+
default_priority = 200
+
+ #: The converter is available or not. Will be filled at the first call of
+ #: the build. The result is shared in the same process.
+ #:
+ #: .. todo:: This should be refactored not to store the state without class
+ #: variable.
available: bool | None = None
+
+ #: A conversion rules the image converter supports.
+ #: It is represented as a list of pair of source image format (mimetype) and
+ #: destination one::
+ #:
+ #: conversion_rules = [
+ #: ('image/svg+xml', 'image/png'),
+ #: ('image/gif', 'image/png'),
+ #: ('application/pdf', 'image/png'),
+ #: ]
conversion_rules: list[tuple[str, str]] = []
- def is_available(self) ->bool:
+ def match(self, node: nodes.image) -> bool:
+ if not self.app.builder.supported_image_types:
+ return False
+ if '?' in node['candidates']:
+ return False
+ if set(self.guess_mimetypes(node)) & set(self.app.builder.supported_image_types):
+ # builder supports the image; no need to convert
+ return False
+ if self.available is None:
+ # store the value to the class variable to share it during the build
+ self.__class__.available = self.is_available()
+
+ if not self.available:
+ return False
+ else:
+ try:
+ self.get_conversion_rule(node)
+ except ValueError:
+ return False
+ else:
+ return True
+
+ def get_conversion_rule(self, node: nodes.image) -> tuple[str, str]:
+ for candidate in self.guess_mimetypes(node):
+ for supported in self.app.builder.supported_image_types:
+ rule = (candidate, supported)
+ if rule in self.conversion_rules:
+ return rule
+
+ msg = 'No conversion rule found'
+ raise ValueError(msg)
+
+ def is_available(self) -> bool:
"""Return the image converter is available or not."""
- pass
+ raise NotImplementedError
+
+ def guess_mimetypes(self, node: nodes.image) -> list[str]:
+ # The special key ? is set for nonlocal URIs.
+ if '?' in node['candidates']:
+ return []
+ elif '*' in node['candidates']:
+ path = os.path.join(self.app.srcdir, node['uri'])
+ guessed = guess_mimetype(path)
+ return [guessed] if guessed is not None else []
+ else:
+ return node['candidates'].keys()
+
+ def handle(self, node: nodes.image) -> None:
+ _from, _to = self.get_conversion_rule(node)
+
+ if _from in node['candidates']:
+ srcpath = node['candidates'][_from]
+ else:
+ srcpath = node['candidates']['*']
+
+ filename = self.env.images[srcpath][1]
+ filename = get_filename_for(filename, _to)
+ ensuredir(self.imagedir)
+ destpath = os.path.join(self.imagedir, filename)
+
+ abs_srcpath = os.path.join(self.app.srcdir, srcpath)
+ if self.convert(abs_srcpath, destpath):
+ if '*' in node['candidates']:
+ node['candidates']['*'] = destpath
+ else:
+ node['candidates'][_to] = destpath
+ node['uri'] = destpath
- def convert(self, _from: str, _to: str) ->bool:
+ self.env.original_image_uri[_StrPath(destpath)] = srcpath
+ self.env.images.add_file(self.env.docname, destpath)
+
+ def convert(self, _from: str, _to: str) -> bool:
"""Convert an image file to the expected format.
*_from* is a path of the source image file, and *_to* is a path
of the destination file.
"""
- pass
+ raise NotImplementedError
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_post_transform(ImageDownloader)
+ app.add_post_transform(DataURIExtractor)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/transforms/references.py b/sphinx/transforms/references.py
index f9e9f641a..6f935aa2a 100644
--- a/sphinx/transforms/references.py
+++ b/sphinx/transforms/references.py
@@ -1,8 +1,13 @@
"""Docutils transforms used by Sphinx."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any
+
from docutils.transforms.references import DanglingReferences
+
from sphinx.transforms import SphinxTransform
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
@@ -11,7 +16,34 @@ if TYPE_CHECKING:
class SphinxDanglingReferences(DanglingReferences):
"""DanglingReferences transform which does not output info messages."""
+ def apply(self, **kwargs: Any) -> None:
+ try:
+ reporter = self.document.reporter
+ report_level = reporter.report_level
+
+ # suppress INFO level messages for a while
+ reporter.report_level = max(reporter.WARNING_LEVEL, reporter.report_level)
+ super().apply() # type: ignore[no-untyped-call]
+ finally:
+ reporter.report_level = report_level
+
class SphinxDomains(SphinxTransform):
"""Collect objects to Sphinx domains for cross references."""
+
default_priority = 850
+
+ def apply(self, **kwargs: Any) -> None:
+ for domain in self.env.domains.values():
+ domain.process_doc(self.env, self.env.docname, self.document)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_transform(SphinxDanglingReferences)
+ app.add_transform(SphinxDomains)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/util/_files.py b/sphinx/util/_files.py
index 9f83a18a5..925200241 100644
--- a/sphinx/util/_files.py
+++ b/sphinx/util/_files.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+
import hashlib
import os.path
from typing import Any
@@ -11,13 +12,39 @@ class FilenameUniqDict(dict[str, tuple[set[str], str]]):
appear in. Used for images and downloadable files in the environment.
"""
- def __init__(self) ->None:
+ def __init__(self) -> None:
self._existing: set[str] = set()
- def __getstate__(self) ->set[str]:
+ def add_file(self, docname: str, newfile: str) -> str:
+ if newfile in self:
+ self[newfile][0].add(docname)
+ return self[newfile][1]
+ uniquename = os.path.basename(newfile)
+ base, ext = os.path.splitext(uniquename)
+ i = 0
+ while uniquename in self._existing:
+ i += 1
+ uniquename = f'{base}{i}{ext}'
+ self[newfile] = ({docname}, uniquename)
+ self._existing.add(uniquename)
+ return uniquename
+
+ def purge_doc(self, docname: str) -> None:
+ for filename, (docs, unique) in list(self.items()):
+ docs.discard(docname)
+ if not docs:
+ del self[filename]
+ self._existing.discard(unique)
+
+ def merge_other(self, docnames: set[str], other: dict[str, tuple[set[str], Any]]) -> None:
+ for filename, (docs, _unique) in other.items():
+ for doc in docs & set(docnames):
+ self.add_file(doc, filename)
+
+ def __getstate__(self) -> set[str]:
return self._existing
- def __setstate__(self, state: set[str]) ->None:
+ def __setstate__(self, state: set[str]) -> None:
self._existing = state
@@ -27,3 +54,23 @@ class DownloadFiles(dict[str, tuple[set[str], str]]):
.. important:: This class would be refactored in nearly future.
Hence don't hack this directly.
"""
+
+ def add_file(self, docname: str, filename: str) -> str:
+ if filename not in self:
+ digest = hashlib.md5(filename.encode(), usedforsecurity=False).hexdigest()
+ dest = f'{digest}/{os.path.basename(filename)}'
+ self[filename] = (set(), dest)
+
+ self[filename][0].add(docname)
+ return self[filename][1]
+
+ def purge_doc(self, docname: str) -> None:
+ for filename, (docs, _dest) in list(self.items()):
+ docs.discard(docname)
+ if not docs:
+ del self[filename]
+
+ def merge_other(self, docnames: set[str], other: dict[str, tuple[set[str], Any]]) -> None:
+ for filename, (docs, _dest) in other.items():
+ for docname in docs & set(docnames):
+ self.add_file(docname, filename)
diff --git a/sphinx/util/_importer.py b/sphinx/util/_importer.py
index 6f88b541b..915750d2d 100644
--- a/sphinx/util/_importer.py
+++ b/sphinx/util/_importer.py
@@ -1,9 +1,27 @@
from __future__ import annotations
+
from importlib import import_module
from typing import Any
+
from sphinx.errors import ExtensionError
-def import_object(object_name: str, /, source: str='') ->Any:
+def import_object(object_name: str, /, source: str = '') -> Any:
"""Import python object by qualname."""
- pass
+ obj_path = object_name.split('.')
+ module_name = obj_path.pop(0)
+ try:
+ obj = import_module(module_name)
+ for name in obj_path:
+ module_name += '.' + name
+ try:
+ obj = getattr(obj, name)
+ except AttributeError:
+ obj = import_module(module_name)
+ except (AttributeError, ImportError) as exc:
+ if source:
+ msg = f'Could not import {object_name} (needed for {source})'
+ raise ExtensionError(msg, exc) from exc
+ msg = f'Could not import {object_name}'
+ raise ExtensionError(msg, exc) from exc
+ return obj
diff --git a/sphinx/util/_io.py b/sphinx/util/_io.py
index 47a4e10e9..3689d9e45 100644
--- a/sphinx/util/_io.py
+++ b/sphinx/util/_io.py
@@ -1,18 +1,34 @@
from __future__ import annotations
+
from typing import TYPE_CHECKING
+
from sphinx.util.console import strip_escape_sequences
+
if TYPE_CHECKING:
from typing import Protocol
-
class SupportsWrite(Protocol):
- pass
+ def write(self, text: str, /) -> int | None:
+ ...
class TeeStripANSI:
"""File-like object writing to two streams."""
- def __init__(self, stream_term: SupportsWrite, stream_file: SupportsWrite
- ) ->None:
+ def __init__(
+ self,
+ stream_term: SupportsWrite,
+ stream_file: SupportsWrite,
+ ) -> None:
self.stream_term = stream_term
self.stream_file = stream_file
+
+ def write(self, text: str, /) -> None:
+ self.stream_term.write(text)
+ self.stream_file.write(strip_escape_sequences(text))
+
+ def flush(self) -> None:
+ if hasattr(self.stream_term, 'flush'):
+ self.stream_term.flush()
+ if hasattr(self.stream_file, 'flush'):
+ self.stream_file.flush()
diff --git a/sphinx/util/_pathlib.py b/sphinx/util/_pathlib.py
index 12641986f..ebbb30607 100644
--- a/sphinx/util/_pathlib.py
+++ b/sphinx/util/_pathlib.py
@@ -11,44 +11,59 @@ or explicit string coercion.
In Sphinx 9, ``Path`` objects will be expected and returned in all instances
that ``_StrPath`` is currently used.
"""
+
from __future__ import annotations
+
import sys
import warnings
from pathlib import Path, PosixPath, PurePath, WindowsPath
from typing import Any
+
from sphinx.deprecation import RemovedInSphinx90Warning
+
_STR_METHODS = frozenset(str.__dict__)
_PATH_NAME = Path().__class__.__name__
+
_MSG = (
- 'Sphinx 9 will drop support for representing paths as strings. Use "pathlib.Path" or "os.fspath" instead.'
- )
-if sys.platform == 'win32':
+ 'Sphinx 9 will drop support for representing paths as strings. '
+ 'Use "pathlib.Path" or "os.fspath" instead.'
+)
+# https://docs.python.org/3/library/stdtypes.html#typesseq-common
+# https://docs.python.org/3/library/stdtypes.html#string-methods
+if sys.platform == 'win32':
class _StrPath(WindowsPath):
+ def replace( # type: ignore[override]
+ self, old: str, new: str, count: int = -1, /,
+ ) -> str:
+ # replace exists in both Path and str;
+ # in Path it makes filesystem changes, so we use the safer str version
+ warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
+ return self.__str__().replace(old, new, count) # NoQA: PLC2801
- def __getattr__(self, item: str) ->Any:
+ def __getattr__(self, item: str) -> Any:
if item in _STR_METHODS:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return getattr(self.__str__(), item)
msg = f'{_PATH_NAME!r} has no attribute {item!r}'
raise AttributeError(msg)
- def __add__(self, other: str) ->str:
+ def __add__(self, other: str) -> str:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return self.__str__() + other
- def __bool__(self) ->bool:
+ def __bool__(self) -> bool:
if not self.__str__():
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return False
return True
- def __contains__(self, item: str) ->bool:
+ def __contains__(self, item: str) -> bool:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return item in self.__str__()
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if isinstance(other, PurePath):
return super().__eq__(other)
if isinstance(other, str):
@@ -56,43 +71,48 @@ if sys.platform == 'win32':
return self.__str__() == other
return NotImplemented
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return super().__hash__()
- def __getitem__(self, item: (int | slice)) ->str:
+ def __getitem__(self, item: int | slice) -> str:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return self.__str__()[item]
- def __len__(self) ->int:
+ def __len__(self) -> int:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return len(self.__str__())
else:
-
-
class _StrPath(PosixPath):
+ def replace( # type: ignore[override]
+ self, old: str, new: str, count: int = -1, /,
+ ) -> str:
+ # replace exists in both Path and str;
+ # in Path it makes filesystem changes, so we use the safer str version
+ warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
+ return self.__str__().replace(old, new, count) # NoQA: PLC2801
- def __getattr__(self, item: str) ->Any:
+ def __getattr__(self, item: str) -> Any:
if item in _STR_METHODS:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return getattr(self.__str__(), item)
msg = f'{_PATH_NAME!r} has no attribute {item!r}'
raise AttributeError(msg)
- def __add__(self, other: str) ->str:
+ def __add__(self, other: str) -> str:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return self.__str__() + other
- def __bool__(self) ->bool:
+ def __bool__(self) -> bool:
if not self.__str__():
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return False
return True
- def __contains__(self, item: str) ->bool:
+ def __contains__(self, item: str) -> bool:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return item in self.__str__()
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if isinstance(other, PurePath):
return super().__eq__(other)
if isinstance(other, str):
@@ -100,13 +120,13 @@ else:
return self.__str__() == other
return NotImplemented
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return super().__hash__()
- def __getitem__(self, item: (int | slice)) ->str:
+ def __getitem__(self, item: int | slice) -> str:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return self.__str__()[item]
- def __len__(self) ->int:
+ def __len__(self) -> int:
warnings.warn(_MSG, RemovedInSphinx90Warning, stacklevel=2)
return len(self.__str__())
diff --git a/sphinx/util/_timestamps.py b/sphinx/util/_timestamps.py
index 9804211d8..32aca5232 100644
--- a/sphinx/util/_timestamps.py
+++ b/sphinx/util/_timestamps.py
@@ -1,10 +1,12 @@
from __future__ import annotations
+
import time
-def _format_rfc3339_microseconds(timestamp: int, /) ->str:
+def _format_rfc3339_microseconds(timestamp: int, /) -> str:
"""Return an RFC 3339 formatted string representing the given timestamp.
:param timestamp: The timestamp to format, in microseconds.
"""
- pass
+ seconds, fraction = divmod(timestamp, 10**6)
+ return time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(seconds)) + f'.{fraction // 1_000}'
diff --git a/sphinx/util/build_phase.py b/sphinx/util/build_phase.py
index 4148d592b..76e94a9b0 100644
--- a/sphinx/util/build_phase.py
+++ b/sphinx/util/build_phase.py
@@ -1,9 +1,11 @@
"""Build phase of Sphinx application."""
+
from enum import IntEnum
class BuildPhase(IntEnum):
"""Build phase of Sphinx application."""
+
INITIALIZATION = 1
READING = 2
CONSISTENCY_CHECK = 3
diff --git a/sphinx/util/cfamily.py b/sphinx/util/cfamily.py
index 93e955260..6c85f8aad 100644
--- a/sphinx/util/cfamily.py
+++ b/sphinx/util/cfamily.py
@@ -1,86 +1,94 @@
"""Utility functions common to the C and C++ domains."""
+
from __future__ import annotations
+
import re
from copy import deepcopy
from typing import TYPE_CHECKING
+
from docutils import nodes
+
from sphinx import addnodes
from sphinx.util import logging
+
if TYPE_CHECKING:
from collections.abc import Callable, Sequence
from typing import Any, TypeAlias
+
from docutils.nodes import TextElement
+
from sphinx.config import Config
+
StringifyTransform: TypeAlias = Callable[[Any], str]
+
logger = logging.getLogger(__name__)
-_whitespace_re = re.compile('\\s+')
-anon_identifier_re = re.compile('(@[a-zA-Z0-9_])[a-zA-Z0-9_]*\\b')
-identifier_re = re.compile(
- """
+
+_whitespace_re = re.compile(r'\s+')
+anon_identifier_re = re.compile(r'(@[a-zA-Z0-9_])[a-zA-Z0-9_]*\b')
+identifier_re = re.compile(r'''
( # This 'extends' _anon_identifier_re with the ordinary identifiers,
# make sure they are in sync.
- (~?\\b[a-zA-Z_]) # ordinary identifiers
+ (~?\b[a-zA-Z_]) # ordinary identifiers
| (@[a-zA-Z0-9_]) # our extension for names of anonymous entities
)
- [a-zA-Z0-9_]*\\b
-"""
- , flags=re.VERBOSE)
-integer_literal_re = re.compile("[1-9][0-9]*(\\'[0-9]+)*")
-octal_literal_re = re.compile("0[0-7]*(\\'[0-7]+)*")
-hex_literal_re = re.compile("0[xX][0-9a-fA-F]+(\\'[0-9a-fA-F]+)*")
-binary_literal_re = re.compile("0[bB][01]+(\\'[01]+)*")
-integers_literal_suffix_re = re.compile(
- """
+ [a-zA-Z0-9_]*\b
+''', flags=re.VERBOSE)
+integer_literal_re = re.compile(r'[1-9][0-9]*(\'[0-9]+)*')
+octal_literal_re = re.compile(r'0[0-7]*(\'[0-7]+)*')
+hex_literal_re = re.compile(r'0[xX][0-9a-fA-F]+(\'[0-9a-fA-F]+)*')
+binary_literal_re = re.compile(r'0[bB][01]+(\'[01]+)*')
+integers_literal_suffix_re = re.compile(r'''
# unsigned and/or (long) long, in any order, but at least one of them
(
([uU] ([lL] | (ll) | (LL))?)
|
(([lL] | (ll) | (LL)) [uU]?)
- )\\b
+ )\b
# the ending word boundary is important for distinguishing
# between suffixes and UDLs in C++
-"""
- , flags=re.VERBOSE)
-float_literal_re = re.compile(
- """
+''', flags=re.VERBOSE)
+float_literal_re = re.compile(r'''
[+-]?(
# decimal
- ([0-9]+(\\'[0-9]+)*[eE][+-]?[0-9]+(\\'[0-9]+)*)
- | (([0-9]+(\\'[0-9]+)*)?\\.[0-9]+(\\'[0-9]+)*([eE][+-]?[0-9]+(\\'[0-9]+)*)?)
- | ([0-9]+(\\'[0-9]+)*\\.([eE][+-]?[0-9]+(\\'[0-9]+)*)?)
+ ([0-9]+(\'[0-9]+)*[eE][+-]?[0-9]+(\'[0-9]+)*)
+ | (([0-9]+(\'[0-9]+)*)?\.[0-9]+(\'[0-9]+)*([eE][+-]?[0-9]+(\'[0-9]+)*)?)
+ | ([0-9]+(\'[0-9]+)*\.([eE][+-]?[0-9]+(\'[0-9]+)*)?)
# hex
- | (0[xX][0-9a-fA-F]+(\\'[0-9a-fA-F]+)*[pP][+-]?[0-9a-fA-F]+(\\'[0-9a-fA-F]+)*)
- | (0[xX]([0-9a-fA-F]+(\\'[0-9a-fA-F]+)*)?\\.
- [0-9a-fA-F]+(\\'[0-9a-fA-F]+)*([pP][+-]?[0-9a-fA-F]+(\\'[0-9a-fA-F]+)*)?)
- | (0[xX][0-9a-fA-F]+(\\'[0-9a-fA-F]+)*\\.([pP][+-]?[0-9a-fA-F]+(\\'[0-9a-fA-F]+)*)?)
+ | (0[xX][0-9a-fA-F]+(\'[0-9a-fA-F]+)*[pP][+-]?[0-9a-fA-F]+(\'[0-9a-fA-F]+)*)
+ | (0[xX]([0-9a-fA-F]+(\'[0-9a-fA-F]+)*)?\.
+ [0-9a-fA-F]+(\'[0-9a-fA-F]+)*([pP][+-]?[0-9a-fA-F]+(\'[0-9a-fA-F]+)*)?)
+ | (0[xX][0-9a-fA-F]+(\'[0-9a-fA-F]+)*\.([pP][+-]?[0-9a-fA-F]+(\'[0-9a-fA-F]+)*)?)
)
-"""
- , flags=re.VERBOSE)
-float_literal_suffix_re = re.compile('[fFlL]\\b')
-char_literal_re = re.compile(
- """
+''', flags=re.VERBOSE)
+float_literal_suffix_re = re.compile(r'[fFlL]\b')
+# the ending word boundary is important for distinguishing between suffixes and UDLs in C++
+char_literal_re = re.compile(r'''
((?:u8)|u|U|L)?
'(
- (?:[^\\\\'])
- | (\\\\(
- (?:['"?\\\\abfnrtv])
+ (?:[^\\'])
+ | (\\(
+ (?:['"?\\abfnrtv])
| (?:[0-7]{1,3})
| (?:x[0-9a-fA-F]{2})
| (?:u[0-9a-fA-F]{4})
| (?:U[0-9a-fA-F]{8})
))
)'
-"""
- , flags=re.VERBOSE)
+''', flags=re.VERBOSE)
+
+
+def verify_description_mode(mode: str) -> None:
+ if mode not in ('lastIsName', 'noneIsName', 'markType', 'markName', 'param', 'udl'):
+ raise Exception("Description mode '%s' is invalid." % mode)
class NoOldIdError(Exception):
+ # Used to avoid implementing unneeded id generation for old id schemes.
pass
class ASTBaseBase:
-
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if type(self) is not type(other):
return NotImplemented
try:
@@ -88,115 +96,174 @@ class ASTBaseBase:
except AttributeError:
return False
- def __str__(self) ->str:
+ def clone(self) -> Any:
+ return deepcopy(self)
+
+ def _stringify(self, transform: StringifyTransform) -> str:
+ raise NotImplementedError(repr(self))
+
+ def __str__(self) -> str:
return self._stringify(str)
- def __repr__(self) ->str:
+ def get_display_string(self) -> str:
+ return self._stringify(lambda ast: ast.get_display_string())
+
+ def __repr__(self) -> str:
return f'<{self.__class__.__name__}: {self._stringify(repr)}>'
+################################################################################
+# Attributes
+################################################################################
+
class ASTAttribute(ASTBaseBase):
- pass
+ def describe_signature(self, signode: TextElement) -> None:
+ raise NotImplementedError(repr(self))
class ASTCPPAttribute(ASTAttribute):
-
- def __init__(self, arg: str) ->None:
+ def __init__(self, arg: str) -> None:
self.arg = arg
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTCPPAttribute):
return NotImplemented
return self.arg == other.arg
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.arg)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return f"[[{self.arg}]]"
-class ASTGnuAttribute(ASTBaseBase):
+ def describe_signature(self, signode: TextElement) -> None:
+ signode.append(addnodes.desc_sig_punctuation('[[', '[['))
+ signode.append(nodes.Text(self.arg))
+ signode.append(addnodes.desc_sig_punctuation(']]', ']]'))
- def __init__(self, name: str, args: (ASTBaseParenExprList | None)) ->None:
+
+class ASTGnuAttribute(ASTBaseBase):
+ def __init__(self, name: str, args: ASTBaseParenExprList | None) -> None:
self.name = name
self.args = args
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTGnuAttribute):
return NotImplemented
return self.name == other.name and self.args == other.args
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.name, self.args))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ if self.args:
+ return self.name + transform(self.args)
+ return self.name
-class ASTGnuAttributeList(ASTAttribute):
- def __init__(self, attrs: list[ASTGnuAttribute]) ->None:
+class ASTGnuAttributeList(ASTAttribute):
+ def __init__(self, attrs: list[ASTGnuAttribute]) -> None:
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTGnuAttributeList):
return NotImplemented
return self.attrs == other.attrs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.attrs)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ attrs = ', '.join(map(transform, self.attrs))
+ return f'__attribute__(({attrs}))'
+
+ def describe_signature(self, signode: TextElement) -> None:
+ signode.append(nodes.Text(str(self)))
+
class ASTIdAttribute(ASTAttribute):
"""For simple attributes defined by the user."""
- def __init__(self, id: str) ->None:
+ def __init__(self, id: str) -> None:
self.id = id
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTIdAttribute):
return NotImplemented
return self.id == other.id
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.id)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return self.id
+
+ def describe_signature(self, signode: TextElement) -> None:
+ signode.append(nodes.Text(self.id))
+
class ASTParenAttribute(ASTAttribute):
"""For paren attributes defined by the user."""
- def __init__(self, id: str, arg: str) ->None:
+ def __init__(self, id: str, arg: str) -> None:
self.id = id
self.arg = arg
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTParenAttribute):
return NotImplemented
return self.id == other.id and self.arg == other.arg
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.id, self.arg))
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return f'{self.id}({self.arg})'
+
+ def describe_signature(self, signode: TextElement) -> None:
+ signode.append(nodes.Text(str(self)))
-class ASTAttributeList(ASTBaseBase):
- def __init__(self, attrs: list[ASTAttribute]) ->None:
+class ASTAttributeList(ASTBaseBase):
+ def __init__(self, attrs: list[ASTAttribute]) -> None:
self.attrs = attrs
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
if not isinstance(other, ASTAttributeList):
return NotImplemented
return self.attrs == other.attrs
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.attrs)
- def __len__(self) ->int:
+ def __len__(self) -> int:
return len(self.attrs)
- def __add__(self, other: ASTAttributeList) ->ASTAttributeList:
+ def __add__(self, other: ASTAttributeList) -> ASTAttributeList:
return ASTAttributeList(self.attrs + other.attrs)
+ def _stringify(self, transform: StringifyTransform) -> str:
+ return ' '.join(map(transform, self.attrs))
+
+ def describe_signature(self, signode: TextElement) -> None:
+ if len(self.attrs) == 0:
+ return
+ self.attrs[0].describe_signature(signode)
+ if len(self.attrs) == 1:
+ return
+ for attr in self.attrs[1:]:
+ signode.append(addnodes.desc_sig_space())
+ attr.describe_signature(signode)
+
+
+################################################################################
class ASTBaseParenExprList(ASTBaseBase):
pass
+################################################################################
+
class UnsupportedMultiCharacterCharLiteral(Exception):
pass
@@ -206,15 +273,222 @@ class DefinitionError(Exception):
class BaseParser:
-
- def __init__(self, definition: str, *, location: (nodes.Node | tuple[
- str, int] | str), config: Config) ->None:
+ def __init__(self, definition: str, *,
+ location: nodes.Node | tuple[str, int] | str,
+ config: Config) -> None:
self.definition = definition.strip()
- self.location = location
+ self.location = location # for warnings
self.config = config
+
self.pos = 0
self.end = len(self.definition)
self.last_match: re.Match[str] | None = None
self._previous_state: tuple[int, re.Match[str] | None] = (0, None)
self.otherErrors: list[DefinitionError] = []
+
+ # in our tests the following is set to False to capture bad parsing
self.allowFallbackExpressionParsing = True
+
+ def _make_multi_error(self, errors: list[Any], header: str) -> DefinitionError:
+ if len(errors) == 1:
+ if len(header) > 0:
+ return DefinitionError(header + '\n' + str(errors[0][0]))
+ else:
+ return DefinitionError(str(errors[0][0]))
+ result = [header, '\n']
+ for e in errors:
+ if len(e[1]) > 0:
+ indent = ' '
+ result.extend((e[1], ':\n'))
+ for line in str(e[0]).split('\n'):
+ if len(line) == 0:
+ continue
+ result.extend((indent, line, '\n'))
+ else:
+ result.append(str(e[0]))
+ return DefinitionError(''.join(result))
+
+ @property
+ def language(self) -> str:
+ raise NotImplementedError
+
+ def status(self, msg: str) -> None:
+ # for debugging
+ indicator = '-' * self.pos + '^'
+ logger.debug(f"{msg}\n{self.definition}\n{indicator}") # NoQA: G004
+
+ def fail(self, msg: str) -> None:
+ errors = []
+ indicator = '-' * self.pos + '^'
+ exMain = DefinitionError(
+ 'Invalid %s declaration: %s [error at %d]\n %s\n %s' %
+ (self.language, msg, self.pos, self.definition, indicator))
+ errors.append((exMain, "Main error"))
+ errors.extend((err, "Potential other error") for err in self.otherErrors)
+ self.otherErrors = []
+ raise self._make_multi_error(errors, '')
+
+ def warn(self, msg: str) -> None:
+ logger.warning(msg, location=self.location)
+
+ def match(self, regex: re.Pattern[str]) -> bool:
+ match = regex.match(self.definition, self.pos)
+ if match is not None:
+ self._previous_state = (self.pos, self.last_match)
+ self.pos = match.end()
+ self.last_match = match
+ return True
+ return False
+
+ def skip_string(self, string: str) -> bool:
+ strlen = len(string)
+ if self.definition[self.pos:self.pos + strlen] == string:
+ self.pos += strlen
+ return True
+ return False
+
+ def skip_word(self, word: str) -> bool:
+ return self.match(re.compile(r'\b%s\b' % re.escape(word)))
+
+ def skip_ws(self) -> bool:
+ return self.match(_whitespace_re)
+
+ def skip_word_and_ws(self, word: str) -> bool:
+ if self.skip_word(word):
+ self.skip_ws()
+ return True
+ return False
+
+ def skip_string_and_ws(self, string: str) -> bool:
+ if self.skip_string(string):
+ self.skip_ws()
+ return True
+ return False
+
+ @property
+ def eof(self) -> bool:
+ return self.pos >= self.end
+
+ @property
+ def current_char(self) -> str:
+ try:
+ return self.definition[self.pos]
+ except IndexError:
+ return 'EOF'
+
+ @property
+ def matched_text(self) -> str:
+ if self.last_match is not None:
+ return self.last_match.group()
+ return ''
+
+ def read_rest(self) -> str:
+ rv = self.definition[self.pos:]
+ self.pos = self.end
+ return rv
+
+ def assert_end(self, *, allowSemicolon: bool = False) -> None:
+ self.skip_ws()
+ if allowSemicolon:
+ if not self.eof and self.definition[self.pos:] != ';':
+ self.fail('Expected end of definition or ;.')
+ else:
+ if not self.eof:
+ self.fail('Expected end of definition.')
+
+ ################################################################################
+
+ @property
+ def id_attributes(self) -> Sequence[str]:
+ raise NotImplementedError
+
+ @property
+ def paren_attributes(self) -> Sequence[str]:
+ raise NotImplementedError
+
+ def _parse_balanced_token_seq(self, end: list[str]) -> str:
+ # TODO: add handling of string literals and similar
+ brackets = {'(': ')', '[': ']', '{': '}'}
+ startPos = self.pos
+ symbols: list[str] = []
+ while not self.eof:
+ if len(symbols) == 0 and self.current_char in end:
+ break
+ if self.current_char in brackets:
+ symbols.append(brackets[self.current_char])
+ elif len(symbols) > 0 and self.current_char == symbols[-1]:
+ symbols.pop()
+ elif self.current_char in ")]}":
+ self.fail("Unexpected '%s' in balanced-token-seq." % self.current_char)
+ self.pos += 1
+ if self.eof:
+ self.fail("Could not find end of balanced-token-seq starting at %d."
+ % startPos)
+ return self.definition[startPos:self.pos]
+
+ def _parse_attribute(self) -> ASTAttribute | None:
+ self.skip_ws()
+ # try C++11 style
+ startPos = self.pos
+ if self.skip_string_and_ws('['):
+ if not self.skip_string('['):
+ self.pos = startPos
+ else:
+ # TODO: actually implement the correct grammar
+ arg = self._parse_balanced_token_seq(end=[']'])
+ if not self.skip_string_and_ws(']'):
+ self.fail("Expected ']' in end of attribute.")
+ if not self.skip_string_and_ws(']'):
+ self.fail("Expected ']' in end of attribute after [[...]")
+ return ASTCPPAttribute(arg)
+
+ # try GNU style
+ if self.skip_word_and_ws('__attribute__'):
+ if not self.skip_string_and_ws('('):
+ self.fail("Expected '(' after '__attribute__'.")
+ if not self.skip_string_and_ws('('):
+ self.fail("Expected '(' after '__attribute__('.")
+ attrs = []
+ while 1:
+ if self.match(identifier_re):
+ name = self.matched_text
+ exprs = self._parse_paren_expression_list()
+ attrs.append(ASTGnuAttribute(name, exprs))
+ if self.skip_string_and_ws(','):
+ continue
+ if self.skip_string_and_ws(')'):
+ break
+ self.fail("Expected identifier, ')', or ',' in __attribute__.")
+ if not self.skip_string_and_ws(')'):
+ self.fail("Expected ')' after '__attribute__((...)'")
+ return ASTGnuAttributeList(attrs)
+
+ # try the simple id attributes defined by the user
+ for id in self.id_attributes:
+ if self.skip_word_and_ws(id):
+ return ASTIdAttribute(id)
+
+ # try the paren attributes defined by the user
+ for id in self.paren_attributes:
+ if not self.skip_string_and_ws(id):
+ continue
+ if not self.skip_string('('):
+ self.fail("Expected '(' after user-defined paren-attribute.")
+ arg = self._parse_balanced_token_seq(end=[')'])
+ if not self.skip_string(')'):
+ self.fail("Expected ')' to end user-defined paren-attribute.")
+ return ASTParenAttribute(id, arg)
+
+ return None
+
+ def _parse_attribute_list(self) -> ASTAttributeList:
+ res = []
+ while True:
+ attr = self._parse_attribute()
+ if attr is None:
+ break
+ res.append(attr)
+ return ASTAttributeList(res)
+
+ def _parse_paren_expression_list(self) -> ASTBaseParenExprList | None:
+ raise NotImplementedError
diff --git a/sphinx/util/console.py b/sphinx/util/console.py
index 1da057880..2b24715ca 100644
--- a/sphinx/util/console.py
+++ b/sphinx/util/console.py
@@ -1,49 +1,140 @@
"""Format colored console output."""
+
from __future__ import annotations
+
import os
import re
import shutil
import sys
from typing import TYPE_CHECKING
+
if TYPE_CHECKING:
from typing import Final
+
+ # fmt: off
+ def reset(text: str) -> str: ... # NoQA: E704
+ def bold(text: str) -> str: ... # NoQA: E704
+ def faint(text: str) -> str: ... # NoQA: E704
+ def standout(text: str) -> str: ... # NoQA: E704
+ def underline(text: str) -> str: ... # NoQA: E704
+ def blink(text: str) -> str: ... # NoQA: E704
+
+ def black(text: str) -> str: ... # NoQA: E704
+ def white(text: str) -> str: ... # NoQA: E704
+ def red(text: str) -> str: ... # NoQA: E704
+ def green(text: str) -> str: ... # NoQA: E704
+ def yellow(text: str) -> str: ... # NoQA: E704
+ def blue(text: str) -> str: ... # NoQA: E704
+ def fuchsia(text: str) -> str: ... # NoQA: E704
+ def teal(text: str) -> str: ... # NoQA: E704
+
+ def darkgray(text: str) -> str: ... # NoQA: E704
+ def lightgray(text: str) -> str: ... # NoQA: E704
+ def darkred(text: str) -> str: ... # NoQA: E704
+ def darkgreen(text: str) -> str: ... # NoQA: E704
+ def brown(text: str) -> str: ... # NoQA: E704
+ def darkblue(text: str) -> str: ... # NoQA: E704
+ def purple(text: str) -> str: ... # NoQA: E704
+ def turquoise(text: str) -> str: ... # NoQA: E704
+ # fmt: on
+
try:
+ # check if colorama is installed to support color on Windows
import colorama
COLORAMA_AVAILABLE = True
except ImportError:
COLORAMA_AVAILABLE = False
-_CSI: Final[str] = re.escape('\x1b[')
-_ansi_color_re: Final[re.Pattern[str]] = re.compile(
- '\\x1b\\[(?:\\d+;){0,2}\\d*m')
-_ansi_re: Final[re.Pattern[str]] = re.compile(_CSI +
- """
+
+_CSI: Final[str] = re.escape('\x1b[') # 'ESC [': Control Sequence Introducer
+
+# Pattern matching ANSI control sequences containing colors.
+_ansi_color_re: Final[re.Pattern[str]] = re.compile(r'\x1b\[(?:\d+;){0,2}\d*m')
+
+_ansi_re: Final[re.Pattern[str]] = re.compile(
+ _CSI
+ + r"""
(?:
- (?:\\d+;){0,2}\\d*m # ANSI color code ('m' is equivalent to '0m')
+ (?:\d+;){0,2}\d*m # ANSI color code ('m' is equivalent to '0m')
|
[012]?K # ANSI Erase in Line ('K' is equivalent to '0K')
- )"""
- , re.VERBOSE | re.ASCII)
+ )""",
+ re.VERBOSE | re.ASCII,
+)
"""Pattern matching ANSI CSI colors (SGR) and erase line (EL) sequences.
See :func:`strip_escape_sequences` for details.
"""
+
codes: dict[str, str] = {}
-def terminal_safe(s: str) ->str:
+def terminal_safe(s: str) -> str:
"""Safely encode a string for printing to the terminal."""
- pass
+ return s.encode('ascii', 'backslashreplace').decode('ascii')
-def get_terminal_width() ->int:
+def get_terminal_width() -> int:
"""Return the width of the terminal in columns."""
- pass
+ return shutil.get_terminal_size().columns - 1
_tw: int = get_terminal_width()
-def strip_colors(s: str) ->str:
+def term_width_line(text: str) -> str:
+ if not codes:
+ # if no coloring, don't output fancy backspaces
+ return text + '\n'
+ else:
+ # codes are not displayed, this must be taken into account
+ return text.ljust(_tw + len(text) - len(strip_escape_sequences(text))) + '\r'
+
+
+def color_terminal() -> bool:
+ if 'NO_COLOR' in os.environ:
+ return False
+ if sys.platform == 'win32' and COLORAMA_AVAILABLE:
+ colorama.just_fix_windows_console()
+ return True
+ if 'FORCE_COLOR' in os.environ:
+ return True
+ if not hasattr(sys.stdout, 'isatty'):
+ return False
+ if not sys.stdout.isatty():
+ return False
+ if 'COLORTERM' in os.environ:
+ return True
+ term = os.environ.get('TERM', 'dumb').lower()
+ return term in ('xterm', 'linux') or 'color' in term
+
+
+def nocolor() -> None:
+ if sys.platform == 'win32' and COLORAMA_AVAILABLE:
+ colorama.deinit()
+ codes.clear()
+
+
+def coloron() -> None:
+ codes.update(_orig_codes)
+
+
+def colorize(name: str, text: str, input_mode: bool = False) -> str:
+ def escseq(name: str) -> str:
+ # Wrap escape sequence with ``\1`` and ``\2`` to let readline know
+ # it is non-printable characters
+ # ref: https://tiswww.case.edu/php/chet/readline/readline.html
+ #
+ # Note: This hack does not work well in Windows (see #5059)
+ escape = codes.get(name, '')
+ if input_mode and escape and sys.platform != 'win32':
+ return '\1' + escape + '\2'
+ else:
+ return escape
+
+ return escseq(name) + text + escseq('reset')
+
+
+def strip_colors(s: str) -> str:
"""Remove the ANSI color codes in a string *s*.
.. caution::
@@ -53,15 +144,15 @@ def strip_colors(s: str) ->str:
.. seealso:: :func:`strip_escape_sequences`
"""
- pass
+ return _ansi_color_re.sub('', s)
-def strip_escape_sequences(text: str, /) ->str:
- """Remove the ANSI CSI colors and "erase in line" sequences.
+def strip_escape_sequences(text: str, /) -> str:
+ r"""Remove the ANSI CSI colors and "erase in line" sequences.
Other `escape sequences `__ (e.g., VT100-specific functions) are not
supported and only control sequences *natively* known to Sphinx (i.e.,
- colors declared in this module and "erase entire line" (``'\\x1b[2K'``))
+ colors declared in this module and "erase entire line" (``'\x1b[2K'``))
are eliminated by this function.
.. caution::
@@ -76,19 +167,44 @@ def strip_escape_sequences(text: str, /) ->str:
__ https://en.wikipedia.org/wiki/ANSI_escape_code
"""
- pass
+ return _ansi_re.sub('', text)
+
+
+def create_color_func(name: str) -> None:
+ def inner(text: str) -> str:
+ return colorize(name, text)
+
+ globals()[name] = inner
+
+_attrs = {
+ 'reset': '39;49;00m',
+ 'bold': '01m',
+ 'faint': '02m',
+ 'standout': '03m',
+ 'underline': '04m',
+ 'blink': '05m',
+}
-_attrs = {'reset': '39;49;00m', 'bold': '01m', 'faint': '02m', 'standout':
- '03m', 'underline': '04m', 'blink': '05m'}
for __name, __value in _attrs.items():
codes[__name] = '\x1b[' + __value
-_colors = [('black', 'darkgray'), ('darkred', 'red'), ('darkgreen', 'green'
- ), ('brown', 'yellow'), ('darkblue', 'blue'), ('purple', 'fuchsia'), (
- 'turquoise', 'teal'), ('lightgray', 'white')]
+
+_colors = [
+ ('black', 'darkgray'),
+ ('darkred', 'red'),
+ ('darkgreen', 'green'),
+ ('brown', 'yellow'),
+ ('darkblue', 'blue'),
+ ('purple', 'fuchsia'),
+ ('turquoise', 'teal'),
+ ('lightgray', 'white'),
+]
+
for __i, (__dark, __light) in enumerate(_colors, 30):
codes[__dark] = '\x1b[%im' % __i
codes[__light] = '\x1b[%im' % (__i + 60)
+
_orig_codes = codes.copy()
+
for _name in codes:
create_color_func(_name)
diff --git a/sphinx/util/display.py b/sphinx/util/display.py
index cdbaa5a8c..f3aea633e 100644
--- a/sphinx/util/display.py
+++ b/sphinx/util/display.py
@@ -1,35 +1,84 @@
from __future__ import annotations
+
import functools
+
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.console import bold, color_terminal
+
if False:
from collections.abc import Callable, Iterable, Iterator
from types import TracebackType
from typing import Any, TypeVar
+
from typing_extensions import ParamSpec
+
T = TypeVar('T')
P = ParamSpec('P')
R = TypeVar('R')
+
logger = logging.getLogger(__name__)
+def display_chunk(chunk: Any) -> str:
+ if isinstance(chunk, list | tuple):
+ if len(chunk) == 1:
+ return str(chunk[0])
+ return f'{chunk[0]} .. {chunk[-1]}'
+ return str(chunk)
+
+
+def status_iterator(
+ iterable: Iterable[T],
+ summary: str,
+ color: str = 'darkgreen',
+ length: int = 0,
+ verbosity: int = 0,
+ stringify_func: Callable[[Any], str] = display_chunk,
+) -> Iterator[T]:
+ # printing on a single line requires ANSI control sequences
+ single_line = verbosity < 1 and color_terminal()
+ bold_summary = bold(summary)
+ if length == 0:
+ logger.info(bold_summary, nonl=True)
+ for item in iterable:
+ logger.info(stringify_func(item) + ' ', nonl=True, color=color)
+ yield item
+ else:
+ for i, item in enumerate(iterable, start=1):
+ if single_line:
+ # clear the entire line ('Erase in Line')
+ logger.info('\x1b[2K', nonl=True)
+ logger.info(f'{bold_summary}[{i / length: >4.0%}] ', nonl=True) # NoQA: G004
+ # Emit the string representation of ``item``
+ logger.info(stringify_func(item), nonl=True, color=color)
+ # If in single-line mode, emit a carriage return to move the cursor
+ # to the start of the line.
+ # If not, emit a newline to move the cursor to the next line.
+ logger.info('\r' * single_line, nonl=single_line)
+ yield item
+ logger.info('')
+
+
class SkipProgressMessage(Exception):
pass
class progress_message:
-
- def __init__(self, message: str, *, nonl: bool=True) ->None:
+ def __init__(self, message: str, *, nonl: bool = True) -> None:
self.message = message
self.nonl = nonl
- def __enter__(self) ->None:
+ def __enter__(self) -> None:
logger.info(bold(self.message + '... '), nonl=self.nonl)
- def __exit__(self, typ: (type[BaseException] | None), val: (
- BaseException | None), tb: (TracebackType | None)) ->bool:
- prefix = '' if self.nonl else bold(self.message + ': ')
+ def __exit__(
+ self,
+ typ: type[BaseException] | None,
+ val: BaseException | None,
+ tb: TracebackType | None,
+ ) -> bool:
+ prefix = "" if self.nonl else bold(self.message + ': ')
if isinstance(val, SkipProgressMessage):
logger.info(prefix + __('skipped'))
if val.args:
@@ -39,12 +88,13 @@ class progress_message:
logger.info(prefix + __('failed'))
else:
logger.info(prefix + __('done'))
- return False
- def __call__(self, f: Callable[P, R]) ->Callable[P, R]:
+ return False
+ def __call__(self, f: Callable[P, R]) -> Callable[P, R]:
@functools.wraps(f)
- def wrapper(*args: P.args, **kwargs: P.kwargs) ->R:
+ def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: # type: ignore[return]
with self:
return f(*args, **kwargs)
+
return wrapper
diff --git a/sphinx/util/docfields.py b/sphinx/util/docfields.py
index 71ce35c34..0ef44d2fd 100644
--- a/sphinx/util/docfields.py
+++ b/sphinx/util/docfields.py
@@ -4,25 +4,37 @@
be domain-specifically transformed to a more appealing presentation.
"""
from __future__ import annotations
+
import contextlib
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes
from docutils.nodes import Element, Node
+
from sphinx import addnodes
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.nodes import get_node_line
+
if TYPE_CHECKING:
from docutils.parsers.rst.states import Inliner
+
from sphinx.directives import ObjectDescription
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import TextlikeNode
+
logger = logging.getLogger(__name__)
-def _is_single_paragraph(node: nodes.field_body) ->bool:
+def _is_single_paragraph(node: nodes.field_body) -> bool:
"""True if the node only contains one paragraph (and system messages)."""
- pass
+ if len(node) == 0:
+ return False
+ elif len(node) > 1:
+ for subnode in node[1:]: # type: Node
+ if not isinstance(subnode, nodes.system_message):
+ return False
+ return isinstance(node[0], nodes.paragraph)
class Field:
@@ -38,11 +50,19 @@ class Field:
:returns: description of the return value
:rtype: description of the return type
"""
+
is_grouped = False
is_typed = False
- def __init__(self, name: str, names: tuple[str, ...]=(), label: str='',
- has_arg: bool=True, rolename: str='', bodyrolename: str='') ->None:
+ def __init__(
+ self,
+ name: str,
+ names: tuple[str, ...] = (),
+ label: str = '',
+ has_arg: bool = True,
+ rolename: str = '',
+ bodyrolename: str = '',
+ ) -> None:
self.name = name
self.names = names
self.label = label
@@ -50,6 +70,73 @@ class Field:
self.rolename = rolename
self.bodyrolename = bodyrolename
+ def make_xref(self, rolename: str, domain: str, target: str,
+ innernode: type[TextlikeNode] = addnodes.literal_emphasis,
+ contnode: Node | None = None, env: BuildEnvironment | None = None,
+ inliner: Inliner | None = None, location: Element | None = None) -> Node:
+ # note: for backwards compatibility env is last, but not optional
+ assert env is not None
+ assert (inliner is None) == (location is None), (inliner, location)
+ if not rolename:
+ return contnode or innernode(target, target) # type: ignore[call-arg]
+ # The domain is passed from DocFieldTransformer. So it surely exists.
+ # So we don't need to take care the env.get_domain() raises an exception.
+ role = env.get_domain(domain).role(rolename)
+ if role is None or inliner is None:
+ if role is None and inliner is not None:
+ msg = __("Problem in %s domain: field is supposed "
+ "to use role '%s', but that role is not in the domain.")
+ logger.warning(__(msg), domain, rolename, location=location)
+ refnode = addnodes.pending_xref('', refdomain=domain, refexplicit=False,
+ reftype=rolename, reftarget=target)
+ refnode += contnode or innernode(target, target) # type: ignore[call-arg]
+ env.get_domain(domain).process_field_xref(refnode)
+ return refnode
+ lineno = -1
+ if location is not None:
+ with contextlib.suppress(ValueError):
+ lineno = get_node_line(location)
+ ns, messages = role(rolename, target, target, lineno, inliner, {}, [])
+ return nodes.inline(target, '', *ns)
+
+ def make_xrefs(self, rolename: str, domain: str, target: str,
+ innernode: type[TextlikeNode] = addnodes.literal_emphasis,
+ contnode: Node | None = None, env: BuildEnvironment | None = None,
+ inliner: Inliner | None = None, location: Element | None = None,
+ ) -> list[Node]:
+ return [self.make_xref(rolename, domain, target, innernode, contnode,
+ env, inliner, location)]
+
+ def make_entry(self, fieldarg: str, content: list[Node]) -> tuple[str, list[Node]]:
+ return (fieldarg, content)
+
+ def make_field(
+ self,
+ types: dict[str, list[Node]],
+ domain: str,
+ item: tuple,
+ env: BuildEnvironment | None = None,
+ inliner: Inliner | None = None,
+ location: Element | None = None,
+ ) -> nodes.field:
+ fieldarg, content = item
+ fieldname = nodes.field_name('', self.label)
+ if fieldarg:
+ fieldname += nodes.Text(' ')
+ fieldname.extend(self.make_xrefs(self.rolename, domain,
+ fieldarg, nodes.Text,
+ env=env, inliner=inliner, location=location))
+
+ if len(content) == 1 and (
+ isinstance(content[0], nodes.Text) or
+ (isinstance(content[0], nodes.inline) and len(content[0]) == 1 and
+ isinstance(content[0][0], nodes.Text))):
+ content = self.make_xrefs(self.bodyrolename, domain,
+ content[0].astext(), contnode=content[0],
+ env=env, inliner=inliner, location=location)
+ fieldbody = nodes.field_body('', nodes.paragraph('', '', *content))
+ return nodes.field('', fieldname, fieldbody)
+
class GroupedField(Field):
"""
@@ -64,14 +151,43 @@ class GroupedField(Field):
:raises ErrorClass: description when it is raised
"""
+
is_grouped = True
list_type = nodes.bullet_list
- def __init__(self, name: str, names: tuple[str, ...]=(), label: str='',
- rolename: str='', can_collapse: bool=False) ->None:
+ def __init__(self, name: str, names: tuple[str, ...] = (), label: str = '',
+ rolename: str = '', can_collapse: bool = False) -> None:
super().__init__(name, names, label, True, rolename)
self.can_collapse = can_collapse
+ def make_field(
+ self,
+ types: dict[str, list[Node]],
+ domain: str,
+ items: tuple,
+ env: BuildEnvironment | None = None,
+ inliner: Inliner | None = None,
+ location: Element | None = None,
+ ) -> nodes.field:
+ fieldname = nodes.field_name('', self.label)
+ listnode = self.list_type()
+ for fieldarg, content in items:
+ par = nodes.paragraph()
+ par.extend(self.make_xrefs(self.rolename, domain, fieldarg,
+ addnodes.literal_strong,
+ env=env, inliner=inliner, location=location))
+ par += nodes.Text(' -- ')
+ par += content
+ listnode += nodes.list_item('', par)
+
+ if len(items) == 1 and self.can_collapse:
+ list_item = cast(nodes.list_item, listnode[0])
+ fieldbody = nodes.field_body('', list_item[0])
+ return nodes.field('', fieldname, fieldbody)
+
+ fieldbody = nodes.field_body('', listnode)
+ return nodes.field('', fieldname, fieldbody)
+
class TypedField(GroupedField):
"""
@@ -92,31 +208,205 @@ class TypedField(GroupedField):
:param SomeClass foo: description of parameter foo
"""
+
is_typed = True
- def __init__(self, name: str, names: tuple[str, ...]=(), typenames:
- tuple[str, ...]=(), label: str='', rolename: str='', typerolename:
- str='', can_collapse: bool=False) ->None:
+ def __init__(
+ self,
+ name: str,
+ names: tuple[str, ...] = (),
+ typenames: tuple[str, ...] = (),
+ label: str = '',
+ rolename: str = '',
+ typerolename: str = '',
+ can_collapse: bool = False,
+ ) -> None:
super().__init__(name, names, label, rolename, can_collapse)
self.typenames = typenames
self.typerolename = typerolename
+ def make_field(
+ self,
+ types: dict[str, list[Node]],
+ domain: str,
+ items: tuple,
+ env: BuildEnvironment | None = None,
+ inliner: Inliner | None = None,
+ location: Element | None = None,
+ ) -> nodes.field:
+ def handle_item(fieldarg: str, content: list[Node]) -> nodes.paragraph:
+ par = nodes.paragraph()
+ par.extend(self.make_xrefs(self.rolename, domain, fieldarg,
+ addnodes.literal_strong, env=env))
+ if fieldarg in types:
+ par += nodes.Text(' (')
+ # NOTE: using .pop() here to prevent a single type node to be
+ # inserted twice into the doctree, which leads to
+ # inconsistencies later when references are resolved
+ fieldtype = types.pop(fieldarg)
+ if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text):
+ typename = fieldtype[0].astext()
+ par.extend(self.make_xrefs(self.typerolename, domain, typename,
+ addnodes.literal_emphasis, env=env,
+ inliner=inliner, location=location))
+ else:
+ par += fieldtype
+ par += nodes.Text(')')
+ has_content = any(c.astext().strip() for c in content)
+ if has_content:
+ par += nodes.Text(' -- ')
+ par += content
+ return par
+
+ fieldname = nodes.field_name('', self.label)
+ if len(items) == 1 and self.can_collapse:
+ fieldarg, content = items[0]
+ bodynode: Node = handle_item(fieldarg, content)
+ else:
+ bodynode = self.list_type()
+ for fieldarg, content in items:
+ bodynode += nodes.list_item('', handle_item(fieldarg, content))
+ fieldbody = nodes.field_body('', bodynode)
+ return nodes.field('', fieldname, fieldbody)
+
class DocFieldTransformer:
"""
Transforms field lists in "doc field" syntax into better-looking
equivalents, using the field type definitions given on a domain.
"""
+
typemap: dict[str, tuple[Field, bool]]
- def __init__(self, directive: ObjectDescription) ->None:
+ def __init__(self, directive: ObjectDescription) -> None:
self.directive = directive
+
self.typemap = directive.get_field_type_map()
- def transform_all(self, node: addnodes.desc_content) ->None:
+ def transform_all(self, node: addnodes.desc_content) -> None:
"""Transform all field list children of a node."""
- pass
+ # don't traverse, only handle field lists that are immediate children
+ for child in node:
+ if isinstance(child, nodes.field_list):
+ self.transform(child)
- def transform(self, node: nodes.field_list) ->None:
+ def transform(self, node: nodes.field_list) -> None:
"""Transform a single field list *node*."""
- pass
+ typemap = self.typemap
+
+ entries: list[nodes.field | tuple[Field, Any, Element]] = []
+ groupindices: dict[str, int] = {}
+ types: dict[str, dict] = {}
+
+ # step 1: traverse all fields and collect field types and content
+ for field in cast(list[nodes.field], node):
+ assert len(field) == 2
+ field_name = cast(nodes.field_name, field[0])
+ field_body = cast(nodes.field_body, field[1])
+ try:
+ # split into field type and argument
+ fieldtype_name, fieldarg = field_name.astext().split(None, 1)
+ except ValueError:
+ # maybe an argument-less field type?
+ fieldtype_name, fieldarg = field_name.astext(), ''
+ typedesc, is_typefield = typemap.get(fieldtype_name, (None, None))
+
+ # collect the content, trying not to keep unnecessary paragraphs
+ if _is_single_paragraph(field_body):
+ paragraph = cast(nodes.paragraph, field_body[0])
+ content = paragraph.children
+ else:
+ content = field_body.children
+
+ # sort out unknown fields
+ if typedesc is None or typedesc.has_arg != bool(fieldarg):
+ # either the field name is unknown, or the argument doesn't
+ # match the spec; capitalize field name and be done with it
+ new_fieldname = fieldtype_name[0:1].upper() + fieldtype_name[1:]
+ if fieldarg:
+ new_fieldname += ' ' + fieldarg
+ field_name[0] = nodes.Text(new_fieldname)
+ entries.append(field)
+
+ # but if this has a type then we can at least link it
+ if (typedesc and is_typefield and content and
+ len(content) == 1 and isinstance(content[0], nodes.Text)):
+ typed_field = cast(TypedField, typedesc)
+ target = content[0].astext()
+ xrefs = typed_field.make_xrefs(
+ typed_field.typerolename,
+ self.directive.domain or '',
+ target,
+ contnode=content[0],
+ env=self.directive.state.document.settings.env,
+ )
+ if _is_single_paragraph(field_body):
+ paragraph = cast(nodes.paragraph, field_body[0])
+ paragraph.clear()
+ paragraph.extend(xrefs)
+ else:
+ field_body.clear()
+ field_body += nodes.paragraph('', '', *xrefs)
+
+ continue
+
+ typename = typedesc.name
+
+ # if the field specifies a type, put it in the types collection
+ if is_typefield:
+ # filter out only inline nodes; others will result in invalid
+ # markup being written out
+ content = [n for n in content if isinstance(n, nodes.Inline | nodes.Text)]
+ if content:
+ types.setdefault(typename, {})[fieldarg] = content
+ continue
+
+ # also support syntax like ``:param type name:``
+ if typedesc.is_typed:
+ try:
+ argtype, argname = fieldarg.rsplit(None, 1)
+ except ValueError:
+ pass
+ else:
+ types.setdefault(typename, {})[argname] = \
+ [nodes.Text(argtype)]
+ fieldarg = argname
+
+ translatable_content = nodes.inline(field_body.rawsource,
+ translatable=True)
+ translatable_content.document = field_body.parent.document
+ translatable_content.source = field_body.parent.source
+ translatable_content.line = field_body.parent.line
+ translatable_content += content
+
+ # grouped entries need to be collected in one entry, while others
+ # get one entry per field
+ if typedesc.is_grouped:
+ if typename in groupindices:
+ group = cast(tuple[Field, list, Node], entries[groupindices[typename]])
+ else:
+ groupindices[typename] = len(entries)
+ group = (typedesc, [], field)
+ entries.append(group)
+ new_entry = typedesc.make_entry(fieldarg, [translatable_content])
+ group[1].append(new_entry)
+ else:
+ new_entry = typedesc.make_entry(fieldarg, [translatable_content])
+ entries.append((typedesc, new_entry, field))
+
+ # step 2: all entries are collected, construct the new field list
+ new_list = nodes.field_list()
+ for entry in entries:
+ if isinstance(entry, nodes.field):
+ # pass-through old field
+ new_list += entry
+ else:
+ fieldtype, items, location = entry
+ fieldtypes = types.get(fieldtype.name, {})
+ env = self.directive.state.document.settings.env
+ inliner = self.directive.state.inliner
+ domain = self.directive.domain or ''
+ new_list += fieldtype.make_field(fieldtypes, domain, items,
+ env=env, inliner=inliner, location=location)
+
+ node.replace_self(new_list)
diff --git a/sphinx/util/docstrings.py b/sphinx/util/docstrings.py
index 8e3c68d14..6ccc5389b 100644
--- a/sphinx/util/docstrings.py
+++ b/sphinx/util/docstrings.py
@@ -1,17 +1,45 @@
"""Utilities for docstring processing."""
+
from __future__ import annotations
+
import re
import sys
+
from docutils.parsers.rst.states import Body
+
field_list_item_re = re.compile(Body.patterns['field_marker'])
-def separate_metadata(s: (str | None)) ->tuple[str | None, dict[str, str]]:
+def separate_metadata(s: str | None) -> tuple[str | None, dict[str, str]]:
"""Separate docstring into metadata and others."""
- pass
+ in_other_element = False
+ metadata: dict[str, str] = {}
+ lines = []
+
+ if not s:
+ return s, metadata
+
+ for line in prepare_docstring(s):
+ if line.strip() == '':
+ in_other_element = False
+ lines.append(line)
+ else:
+ matched = field_list_item_re.match(line)
+ if matched and not in_other_element:
+ field_name = matched.group()[1:].split(':', 1)[0]
+ if field_name.startswith('meta '):
+ name = field_name[5:].strip()
+ metadata[name] = line[matched.end():].strip()
+ else:
+ lines.append(line)
+ else:
+ in_other_element = True
+ lines.append(line)
+
+ return '\n'.join(lines), metadata
-def prepare_docstring(s: str, tabsize: int=8) ->list[str]:
+def prepare_docstring(s: str, tabsize: int = 8) -> list[str]:
"""Convert a docstring into lines of parseable reST. Remove common leading
indentation, where the indentation of the first line is ignored.
@@ -19,11 +47,42 @@ def prepare_docstring(s: str, tabsize: int=8) ->list[str]:
ViewList (used as argument of nested_parse().) An empty line is added to
act as a separator between this docstring and following content.
"""
- pass
+ lines = s.expandtabs(tabsize).splitlines()
+ # Find minimum indentation of any non-blank lines after ignored lines.
+ margin = sys.maxsize
+ for line in lines[1:]:
+ content = len(line.lstrip())
+ if content:
+ indent = len(line) - content
+ margin = min(margin, indent)
+ # Remove indentation from the first line.
+ if len(lines):
+ lines[0] = lines[0].lstrip()
+ if margin < sys.maxsize:
+ for i in range(1, len(lines)):
+ lines[i] = lines[i][margin:]
+ # Remove any leading blank lines.
+ while lines and not lines[0]:
+ lines.pop(0)
+ # make sure there is an empty line at the end
+ if lines and lines[-1]:
+ lines.append('')
+ return lines
-def prepare_commentdoc(s: str) ->list[str]:
+def prepare_commentdoc(s: str) -> list[str]:
"""Extract documentation comment lines (starting with #:) and return them
as a list of lines. Returns an empty list if there is no documentation.
"""
- pass
+ result = []
+ lines = [line.strip() for line in s.expandtabs().splitlines()]
+ for line in lines:
+ if line.startswith('#:'):
+ line = line[2:]
+ # the first space after the comment is ignored
+ if line and line[0] == ' ':
+ line = line[1:]
+ result.append(line)
+ if result and result[-1]:
+ result.append('')
+ return result
diff --git a/sphinx/util/docutils.py b/sphinx/util/docutils.py
index d4889e254..269f3794a 100644
--- a/sphinx/util/docutils.py
+++ b/sphinx/util/docutils.py
@@ -1,111 +1,147 @@
"""Utility functions for docutils."""
+
from __future__ import annotations
+
import os
import re
-from collections.abc import Sequence
+from collections.abc import Sequence # NoQA: TCH003
from contextlib import contextmanager
from copy import copy
from os import path
from typing import IO, TYPE_CHECKING, Any, cast
+
import docutils
from docutils import nodes
from docutils.io import FileOutput
from docutils.parsers.rst import Directive, directives, roles
-from docutils.parsers.rst.states import Inliner
+from docutils.parsers.rst.states import Inliner # NoQA: TCH002
from docutils.statemachine import State, StateMachine, StringList
from docutils.utils import Reporter, unescape
+
from sphinx.errors import SphinxError
from sphinx.locale import _, __
from sphinx.util import logging
from sphinx.util.parsing import nested_parse_to_nodes
+
logger = logging.getLogger(__name__)
-report_re = re.compile(
- '^(.+?:(?:\\d+)?): \\((DEBUG|INFO|WARNING|ERROR|SEVERE)/(\\d+)?\\) ')
+report_re = re.compile('^(.+?:(?:\\d+)?): \\((DEBUG|INFO|WARNING|ERROR|SEVERE)/(\\d+)?\\) ')
+
if TYPE_CHECKING:
- from collections.abc import Callable, Iterator
+ from collections.abc import Callable, Iterator # NoQA: TCH003
from types import ModuleType
+
from docutils.frontend import Values
from docutils.nodes import Element, Node, system_message
+
from sphinx.builders import Builder
from sphinx.config import Config
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import RoleFunction
+
+
additional_nodes: set[type[Element]] = set()
@contextmanager
-def docutils_namespace() ->Iterator[None]:
+def docutils_namespace() -> Iterator[None]:
"""Create namespace for reST parsers."""
- pass
+ try:
+ _directives = copy(directives._directives) # type: ignore[attr-defined]
+ _roles = copy(roles._roles) # type: ignore[attr-defined]
+
+ yield
+ finally:
+ directives._directives = _directives # type: ignore[attr-defined]
+ roles._roles = _roles # type: ignore[attr-defined]
+ for node in list(additional_nodes):
+ unregister_node(node)
+ additional_nodes.discard(node)
-def is_directive_registered(name: str) ->bool:
+
+def is_directive_registered(name: str) -> bool:
"""Check the *name* directive is already registered."""
- pass
+ return name in directives._directives # type: ignore[attr-defined]
-def register_directive(name: str, directive: type[Directive]) ->None:
+def register_directive(name: str, directive: type[Directive]) -> None:
"""Register a directive to docutils.
This modifies global state of docutils. So it is better to use this
inside ``docutils_namespace()`` to prevent side-effects.
"""
- pass
+ directives.register_directive(name, directive)
-def is_role_registered(name: str) ->bool:
+def is_role_registered(name: str) -> bool:
"""Check the *name* role is already registered."""
- pass
+ return name in roles._roles # type: ignore[attr-defined]
-def register_role(name: str, role: RoleFunction) ->None:
+def register_role(name: str, role: RoleFunction) -> None:
"""Register a role to docutils.
This modifies global state of docutils. So it is better to use this
inside ``docutils_namespace()`` to prevent side-effects.
"""
- pass
+ roles.register_local_role(name, role) # type: ignore[arg-type]
-def unregister_role(name: str) ->None:
+def unregister_role(name: str) -> None:
"""Unregister a role from docutils."""
- pass
+ roles._roles.pop(name, None) # type: ignore[attr-defined]
-def is_node_registered(node: type[Element]) ->bool:
+def is_node_registered(node: type[Element]) -> bool:
"""Check the *node* is already registered."""
- pass
+ return hasattr(nodes.GenericNodeVisitor, 'visit_' + node.__name__)
-def register_node(node: type[Element]) ->None:
+def register_node(node: type[Element]) -> None:
"""Register a node to docutils.
This modifies global state of some visitors. So it is better to use this
inside ``docutils_namespace()`` to prevent side-effects.
"""
- pass
+ if not hasattr(nodes.GenericNodeVisitor, 'visit_' + node.__name__):
+ nodes._add_node_class_names([node.__name__]) # type: ignore[attr-defined]
+ additional_nodes.add(node)
-def unregister_node(node: type[Element]) ->None:
+def unregister_node(node: type[Element]) -> None:
"""Unregister a node from docutils.
This is inverse of ``nodes._add_nodes_class_names()``.
"""
- pass
+ if hasattr(nodes.GenericNodeVisitor, 'visit_' + node.__name__):
+ delattr(nodes.GenericNodeVisitor, "visit_" + node.__name__)
+ delattr(nodes.GenericNodeVisitor, "depart_" + node.__name__)
+ delattr(nodes.SparseNodeVisitor, 'visit_' + node.__name__)
+ delattr(nodes.SparseNodeVisitor, 'depart_' + node.__name__)
@contextmanager
-def patched_get_language() ->Iterator[None]:
+def patched_get_language() -> Iterator[None]:
"""Patch docutils.languages.get_language() temporarily.
This ignores the second argument ``reporter`` to suppress warnings.
refs: https://github.com/sphinx-doc/sphinx/issues/3788
"""
- pass
+ from docutils.languages import get_language
+
+ def patched_get_language(language_code: str, reporter: Reporter | None = None) -> Any:
+ return get_language(language_code)
+
+ try:
+ docutils.languages.get_language = patched_get_language # type: ignore[assignment]
+ yield
+ finally:
+ # restore original implementations
+ docutils.languages.get_language = get_language
@contextmanager
-def patched_rst_get_language() ->Iterator[None]:
+def patched_rst_get_language() -> Iterator[None]:
"""Patch docutils.parsers.rst.languages.get_language().
Starting from docutils 0.17, get_language() in ``rst.languages``
also has a reporter, which needs to be disabled temporarily.
@@ -115,19 +151,42 @@ def patched_rst_get_language() ->Iterator[None]:
refs: https://github.com/sphinx-doc/sphinx/issues/10179
"""
- pass
+ from docutils.parsers.rst.languages import get_language
+
+ def patched_get_language(language_code: str, reporter: Reporter | None = None) -> Any:
+ return get_language(language_code)
+
+ try:
+ docutils.parsers.rst.languages.get_language = patched_get_language # type: ignore[assignment]
+ yield
+ finally:
+ # restore original implementations
+ docutils.parsers.rst.languages.get_language = get_language
@contextmanager
-def using_user_docutils_conf(confdir: (str | None)) ->Iterator[None]:
+def using_user_docutils_conf(confdir: str | None) -> Iterator[None]:
"""Let docutils know the location of ``docutils.conf`` for Sphinx."""
- pass
+ try:
+ docutilsconfig = os.environ.get('DOCUTILSCONFIG', None)
+ if confdir:
+ os.environ['DOCUTILSCONFIG'] = path.join(path.abspath(confdir), 'docutils.conf')
+
+ yield
+ finally:
+ if docutilsconfig is None:
+ os.environ.pop('DOCUTILSCONFIG', None)
+ else:
+ os.environ['DOCUTILSCONFIG'] = docutilsconfig
@contextmanager
-def patch_docutils(confdir: (str | None)=None) ->Iterator[None]:
+def patch_docutils(confdir: str | None = None) -> Iterator[None]:
"""Patch to docutils temporarily."""
- pass
+ with patched_get_language(), \
+ patched_rst_get_language(), \
+ using_user_docutils_conf(confdir):
+ yield
class CustomReSTDispatcher:
@@ -137,17 +196,40 @@ class CustomReSTDispatcher:
by original one temporarily.
"""
- def __init__(self) ->None:
+ def __init__(self) -> None:
self.directive_func: Callable = lambda *args: (None, [])
self.roles_func: Callable = lambda *args: (None, [])
- def __enter__(self) ->None:
+ def __enter__(self) -> None:
self.enable()
- def __exit__(self, exc_type: type[Exception], exc_value: Exception,
- traceback: Any) ->None:
+ def __exit__(
+ self, exc_type: type[Exception], exc_value: Exception, traceback: Any,
+ ) -> None:
self.disable()
+ def enable(self) -> None:
+ self.directive_func = directives.directive
+ self.role_func = roles.role
+
+ directives.directive = self.directive # type: ignore[assignment]
+ roles.role = self.role # type: ignore[assignment]
+
+ def disable(self) -> None:
+ directives.directive = self.directive_func
+ roles.role = self.role_func
+
+ def directive(self,
+ directive_name: str, language_module: ModuleType, document: nodes.document,
+ ) -> tuple[type[Directive] | None, list[system_message]]:
+ return self.directive_func(directive_name, language_module, document)
+
+ def role(
+ self, role_name: str, language_module: ModuleType, lineno: int, reporter: Reporter,
+ ) -> tuple[RoleFunction, list[system_message]]:
+ return self.role_func(role_name, language_module, # type: ignore[return-value]
+ lineno, reporter)
+
class ElementLookupError(Exception):
pass
@@ -158,58 +240,126 @@ class sphinx_domains(CustomReSTDispatcher):
markup takes precedence.
"""
- def __init__(self, env: BuildEnvironment) ->None:
+ def __init__(self, env: BuildEnvironment) -> None:
self.env = env
super().__init__()
- def lookup_domain_element(self, type: str, name: str) ->Any:
+ def lookup_domain_element(self, type: str, name: str) -> Any:
"""Lookup a markup element (directive or role), given its name which can
be a full name (with domain).
"""
- pass
+ name = name.lower()
+ # explicit domain given?
+ if ':' in name:
+ domain_name, name = name.split(':', 1)
+ if domain_name in self.env.domains:
+ domain = self.env.get_domain(domain_name)
+ element = getattr(domain, type)(name)
+ if element is not None:
+ return element, []
+ else:
+ logger.warning(_('unknown directive or role name: %s:%s'), domain_name, name)
+ # else look in the default domain
+ else:
+ def_domain = self.env.temp_data.get('default_domain')
+ if def_domain is not None:
+ element = getattr(def_domain, type)(name)
+ if element is not None:
+ return element, []
+
+ # always look in the std domain
+ element = getattr(self.env.get_domain('std'), type)(name)
+ if element is not None:
+ return element, []
+
+ raise ElementLookupError
+
+ def directive(self,
+ directive_name: str, language_module: ModuleType, document: nodes.document,
+ ) -> tuple[type[Directive] | None, list[system_message]]:
+ try:
+ return self.lookup_domain_element('directive', directive_name)
+ except ElementLookupError:
+ return super().directive(directive_name, language_module, document)
+
+ def role(
+ self, role_name: str, language_module: ModuleType, lineno: int, reporter: Reporter,
+ ) -> tuple[RoleFunction, list[system_message]]:
+ try:
+ return self.lookup_domain_element('role', role_name)
+ except ElementLookupError:
+ return super().role(role_name, language_module, lineno, reporter)
class WarningStream:
- pass
+ def write(self, text: str) -> None:
+ matched = report_re.search(text)
+ if not matched:
+ logger.warning(text.rstrip("\r\n"), type="docutils")
+ else:
+ location, type, level = matched.groups()
+ message = report_re.sub('', text).rstrip()
+ logger.log(type, message, location=location, type="docutils")
class LoggingReporter(Reporter):
-
@classmethod
- def from_reporter(cls: type[LoggingReporter], reporter: Reporter
- ) ->LoggingReporter:
+ def from_reporter(cls: type[LoggingReporter], reporter: Reporter) -> LoggingReporter:
"""Create an instance of LoggingReporter from other reporter object."""
- pass
+ return cls(reporter.source, reporter.report_level, reporter.halt_level,
+ reporter.debug_flag, reporter.error_handler)
- def __init__(self, source: str, report_level: int=Reporter.
- WARNING_LEVEL, halt_level: int=Reporter.SEVERE_LEVEL, debug: bool=
- False, error_handler: str='backslashreplace') ->None:
+ def __init__(self, source: str, report_level: int = Reporter.WARNING_LEVEL,
+ halt_level: int = Reporter.SEVERE_LEVEL, debug: bool = False,
+ error_handler: str = 'backslashreplace') -> None:
stream = cast(IO, WarningStream())
- super().__init__(source, report_level, halt_level, stream, debug,
- error_handler=error_handler)
+ super().__init__(source, report_level, halt_level,
+ stream, debug, error_handler=error_handler)
class NullReporter(Reporter):
"""A dummy reporter; write nothing."""
- def __init__(self) ->None:
+ def __init__(self) -> None:
super().__init__('', 999, 4)
@contextmanager
-def switch_source_input(state: State, content: StringList) ->Iterator[None]:
+def switch_source_input(state: State, content: StringList) -> Iterator[None]:
"""Switch current source input of state temporarily."""
- pass
+ try:
+ # remember the original ``get_source_and_line()`` method
+ gsal = state.memo.reporter.get_source_and_line # type: ignore[attr-defined]
+
+ # replace it by new one
+ state_machine: StateMachine[None] = StateMachine([], None) # type: ignore[arg-type]
+ state_machine.input_lines = content
+ state.memo.reporter.get_source_and_line = state_machine.get_source_and_line # type: ignore[attr-defined] # NoQA: E501
+
+ yield
+ finally:
+ # restore the method
+ state.memo.reporter.get_source_and_line = gsal # type: ignore[attr-defined]
class SphinxFileOutput(FileOutput):
"""Better FileOutput class for Sphinx."""
- def __init__(self, **kwargs: Any) ->None:
+ def __init__(self, **kwargs: Any) -> None:
self.overwrite_if_changed = kwargs.pop('overwrite_if_changed', False)
kwargs.setdefault('encoding', 'utf-8')
super().__init__(**kwargs)
+ def write(self, data: str) -> str:
+ if (self.destination_path and self.autoclose and 'b' not in self.mode and
+ self.overwrite_if_changed and os.path.exists(self.destination_path)):
+ with open(self.destination_path, encoding=self.encoding) as f:
+ # skip writing: content not changed
+ if f.read() == data:
+ return data
+
+ return super().write(data)
+
class SphinxDirective(Directive):
"""A base class for Sphinx directives.
@@ -223,44 +373,50 @@ class SphinxDirective(Directive):
"""
@property
- def env(self) ->BuildEnvironment:
+ def env(self) -> BuildEnvironment:
"""Reference to the :class:`.BuildEnvironment` object.
.. versionadded:: 1.8
"""
- pass
+ return self.state.document.settings.env
@property
- def config(self) ->Config:
+ def config(self) -> Config:
"""Reference to the :class:`.Config` object.
.. versionadded:: 1.8
"""
- pass
+ return self.env.config
- def get_source_info(self) ->tuple[str, int]:
+ def get_source_info(self) -> tuple[str, int]:
"""Get source and line number.
.. versionadded:: 3.0
"""
- pass
+ return self.state_machine.get_source_and_line(self.lineno)
- def set_source_info(self, node: Node) ->None:
+ def set_source_info(self, node: Node) -> None:
"""Set source and line number to the node.
.. versionadded:: 2.1
"""
- pass
+ node.source, node.line = self.get_source_info()
- def get_location(self) ->str:
+ def get_location(self) -> str:
"""Get current location info for logging.
.. versionadded:: 4.2
"""
- pass
-
- def parse_content_to_nodes(self, allow_section_headings: bool=False
- ) ->list[Node]:
+ source, line = self.get_source_info()
+ if source and line:
+ return f'{source}:{line}'
+ if source:
+ return f'{source}:'
+ if line:
+ return f'<unknown>:{line}'
+ return ''
+
+ def parse_content_to_nodes(self, allow_section_headings: bool = False) -> list[Node]:
"""Parse the directive's content into nodes.
:param allow_section_headings:
@@ -273,10 +429,16 @@ class SphinxDirective(Directive):
.. versionadded:: 7.4
"""
- pass
-
- def parse_text_to_nodes(self, text: str='', /, *, offset: int=-1,
- allow_section_headings: bool=False) ->list[Node]:
+ return nested_parse_to_nodes(
+ self.state,
+ self.content,
+ offset=self.content_offset,
+ allow_section_headings=allow_section_headings,
+ )
+
+ def parse_text_to_nodes(
+ self, text: str = '', /, *, offset: int = -1, allow_section_headings: bool = False,
+ ) -> list[Node]:
"""Parse *text* into nodes.
:param text:
@@ -293,10 +455,18 @@ class SphinxDirective(Directive):
.. versionadded:: 7.4
"""
- pass
-
- def parse_inline(self, text: str, *, lineno: int=-1) ->tuple[list[Node],
- list[system_message]]:
+ if offset == -1:
+ offset = self.content_offset
+ return nested_parse_to_nodes(
+ self.state,
+ text,
+ offset=offset,
+ allow_section_headings=allow_section_headings,
+ )
+
+ def parse_inline(
+ self, text: str, *, lineno: int = -1,
+ ) -> tuple[list[Node], list[system_message]]:
"""Parse *text* as inline elements.
:param text:
@@ -310,7 +480,9 @@ class SphinxDirective(Directive):
.. versionadded:: 7.4
"""
- pass
+ if lineno == -1:
+ lineno = self.lineno
+ return self.state.inline_text(text, lineno)
class SphinxRole:
@@ -323,23 +495,30 @@ class SphinxRole:
.. note:: The subclasses of this class might not work with docutils.
This class is strongly coupled with Sphinx.
"""
- name: str
- rawtext: str
- text: str
- lineno: int
- inliner: Inliner
+
+ name: str #: The role name actually used in the document.
+ rawtext: str #: A string containing the entire interpreted text input.
+ text: str #: The interpreted text content.
+ lineno: int #: The line number where the interpreted text begins.
+ inliner: Inliner #: The ``docutils.parsers.rst.states.Inliner`` object.
+ #: A dictionary of directive options for customisation
+ #: (from the "role" directive).
options: dict[str, Any]
+ #: A list of strings, the directive content for customisation
+ #: (from the "role" directive).
content: Sequence[str]
def __call__(self, name: str, rawtext: str, text: str, lineno: int,
- inliner: Inliner, options: (dict | None)=None, content: Sequence[
- str]=()) ->tuple[list[Node], list[system_message]]:
+ inliner: Inliner, options: dict | None = None, content: Sequence[str] = (),
+ ) -> tuple[list[Node], list[system_message]]:
self.rawtext = rawtext
self.text = unescape(text)
self.lineno = lineno
self.inliner = inliner
self.options = options if options is not None else {}
self.content = content
+
+ # guess role type
if name:
self.name = name.lower()
else:
@@ -349,30 +528,51 @@ class SphinxRole:
if not self.name:
msg = 'cannot determine default role!'
raise SphinxError(msg)
+
return self.run()
+ def run(self) -> tuple[list[Node], list[system_message]]:
+ raise NotImplementedError
+
@property
- def env(self) ->BuildEnvironment:
+ def env(self) -> BuildEnvironment:
"""Reference to the :class:`.BuildEnvironment` object.
.. versionadded:: 2.0
"""
- pass
+ return self.inliner.document.settings.env
@property
- def config(self) ->Config:
+ def config(self) -> Config:
"""Reference to the :class:`.Config` object.
.. versionadded:: 2.0
"""
- pass
+ return self.env.config
- def get_location(self) ->str:
+ def get_source_info(self, lineno: int | None = None) -> tuple[str, int]:
+ # .. versionadded:: 3.0
+ if lineno is None:
+ lineno = self.lineno
+ return self.inliner.reporter.get_source_and_line(lineno) # type: ignore[attr-defined]
+
+ def set_source_info(self, node: Node, lineno: int | None = None) -> None:
+ # .. versionadded:: 2.0
+ node.source, node.line = self.get_source_info(lineno)
+
+ def get_location(self) -> str:
"""Get current location info for logging.
.. versionadded:: 4.2
"""
- pass
+ source, line = self.get_source_info()
+ if source and line:
+ return f'{source}:{line}'
+ if source:
+ return f'{source}:'
+ if line:
+ return f'<unknown>:{line}'
+ return ''
class ReferenceRole(SphinxRole):
@@ -384,18 +584,24 @@ class ReferenceRole(SphinxRole):
.. versionadded:: 2.0
"""
- has_explicit_title: bool
- disabled: bool
- title: str
- target: str
- explicit_title_re = re.compile('^(.+?)\\s*(?<!\\x00)<(.*?)>$', re.DOTALL)
+
+ has_explicit_title: bool #: A boolean indicates the role has explicit title or not.
+ disabled: bool #: A boolean indicates the reference is disabled.
+ title: str #: The link title for the interpreted text.
+ target: str #: The link target for the interpreted text.
+
+ # \x00 means the "<" was backslash-escaped
+ explicit_title_re = re.compile(r'^(.+?)\s*(?<!\x00)<(.*?)>$', re.DOTALL)
def __call__(self, name: str, rawtext: str, text: str, lineno: int,
- inliner: Inliner, options: (dict | None)=None, content: Sequence[
- str]=()) ->tuple[list[Node], list[system_message]]:
+ inliner: Inliner, options: dict | None = None, content: Sequence[str] = (),
+ ) -> tuple[list[Node], list[system_message]]:
if options is None:
options = {}
+
+ # if the first character is a bang, don't cross-reference at all
self.disabled = text.startswith('!')
+
matched = self.explicit_title_re.match(text)
if matched:
self.has_explicit_title = True
@@ -405,8 +611,8 @@ class ReferenceRole(SphinxRole):
self.has_explicit_title = False
self.title = unescape(text)
self.target = unescape(text)
- return super().__call__(name, rawtext, text, lineno, inliner,
- options, content)
+
+ return super().__call__(name, rawtext, text, lineno, inliner, options, content)
class SphinxTranslator(nodes.NodeVisitor):
@@ -423,13 +629,13 @@ class SphinxTranslator(nodes.NodeVisitor):
This class is strongly coupled with Sphinx.
"""
- def __init__(self, document: nodes.document, builder: Builder) ->None:
+ def __init__(self, document: nodes.document, builder: Builder) -> None:
super().__init__(document)
self.builder = builder
self.config = builder.config
self.settings = document.settings
- def dispatch_visit(self, node: Node) ->None:
+ def dispatch_visit(self, node: Node) -> None:
"""
Dispatch node to appropriate visitor method.
The priority of visitor method is:
@@ -438,9 +644,15 @@ class SphinxTranslator(nodes.NodeVisitor):
2. ``self.visit_{super_node_class}()``
3. ``self.unknown_visit()``
"""
- pass
+ for node_class in node.__class__.__mro__:
+ method = getattr(self, 'visit_%s' % (node_class.__name__), None)
+ if method:
+ method(node)
+ break
+ else:
+ super().dispatch_visit(node)
- def dispatch_departure(self, node: Node) ->None:
+ def dispatch_departure(self, node: Node) -> None:
"""
Dispatch node to appropriate departure method.
The priority of departure method is:
@@ -449,17 +661,43 @@ class SphinxTranslator(nodes.NodeVisitor):
2. ``self.depart_{super_node_class}()``
3. ``self.unknown_departure()``
"""
- pass
+ for node_class in node.__class__.__mro__:
+ method = getattr(self, 'depart_%s' % (node_class.__name__), None)
+ if method:
+ method(node)
+ break
+ else:
+ super().dispatch_departure(node)
+
+ def unknown_visit(self, node: Node) -> None:
+ logger.warning(__('unknown node type: %r'), node, location=node)
+# cache a vanilla instance of nodes.document
+# Used in new_document() function
__document_cache__: tuple[Values, Reporter]
-def new_document(source_path: str, settings: Any=None) ->nodes.document:
+def new_document(source_path: str, settings: Any = None) -> nodes.document:
"""Return a new empty document object. This is an alternative of docutils'.
This is a simple wrapper for ``docutils.utils.new_document()``. It
caches the result of docutils' and use it on second call for instantiation.
This makes an instantiation of document nodes much faster.
"""
- pass
+ global __document_cache__
+ try:
+ cached_settings, reporter = __document_cache__
+ except NameError:
+ doc = docutils.utils.new_document(source_path)
+ __document_cache__ = cached_settings, reporter = doc.settings, doc.reporter
+
+ if settings is None:
+ # Make a copy of the cached settings to accelerate instantiation
+ settings = copy(cached_settings)
+
+ # Create a new instance of nodes.document using cached reporter
+ from sphinx import addnodes
+ document = addnodes.document(settings, reporter, source=source_path)
+ document.note_source(source_path, -1)
+ return document
diff --git a/sphinx/util/exceptions.py b/sphinx/util/exceptions.py
index 53e2b545f..577ec734e 100644
--- a/sphinx/util/exceptions.py
+++ b/sphinx/util/exceptions.py
@@ -1,19 +1,68 @@
from __future__ import annotations
+
import sys
import traceback
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING
+
from sphinx.errors import SphinxParallelError
from sphinx.util.console import strip_escape_sequences
+
if TYPE_CHECKING:
from sphinx.application import Sphinx
-def save_traceback(app: (Sphinx | None), exc: BaseException) ->str:
+def save_traceback(app: Sphinx | None, exc: BaseException) -> str:
"""Save the given exception's traceback in a temporary file."""
- pass
+ import platform
+
+ import docutils
+ import jinja2
+ import pygments
+
+ import sphinx
+
+ if isinstance(exc, SphinxParallelError):
+ exc_format = '(Error in parallel process)\n' + exc.traceback
+ else:
+ exc_format = traceback.format_exc()
+
+ if app is None:
+ last_msgs = exts_list = ''
+ else:
+ extensions = app.extensions.values()
+ last_msgs = '\n'.join(f'# {strip_escape_sequences(s).strip()}'
+ for s in app.messagelog)
+ exts_list = '\n'.join(f'# {ext.name} ({ext.version})' for ext in extensions
+ if ext.version != 'builtin')
+
+ with NamedTemporaryFile('w', suffix='.log', prefix='sphinx-err-', delete=False) as f:
+ f.write(f"""\
+# Platform: {sys.platform}; ({platform.platform()})
+# Sphinx version: {sphinx.__display_version__}
+# Python version: {platform.python_version()} ({platform.python_implementation()})
+# Docutils version: {docutils.__version__}
+# Jinja2 version: {jinja2.__version__}
+# Pygments version: {pygments.__version__}
+
+# Last messages:
+{last_msgs}
+
+# Loaded extensions:
+{exts_list}
+
+# Traceback:
+{exc_format}
+""")
+ return f.name
-def format_exception_cut_frames(x: int=1) ->str:
+def format_exception_cut_frames(x: int = 1) -> str:
"""Format an exception with traceback, but only the last x frames."""
- pass
+ typ, val, tb = sys.exc_info()
+ # res = ['Traceback (most recent call last):\n']
+ res: list[str] = []
+ tbres = traceback.format_tb(tb)
+ res += tbres[-x:]
+ res += traceback.format_exception_only(typ, val)
+ return ''.join(res)
diff --git a/sphinx/util/fileutil.py b/sphinx/util/fileutil.py
index fa4bab77f..259a2af19 100644
--- a/sphinx/util/fileutil.py
+++ b/sphinx/util/fileutil.py
@@ -1,30 +1,44 @@
"""File utility functions for Sphinx."""
+
from __future__ import annotations
+
import os
import posixpath
from typing import TYPE_CHECKING, Any
+
from docutils.utils import relative_path
+
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.osutil import copyfile, ensuredir
+
if TYPE_CHECKING:
from collections.abc import Callable
+
from sphinx.util.template import BaseRenderer
from sphinx.util.typing import PathMatcher
+
logger = logging.getLogger(__name__)
-def _template_basename(filename: (str | os.PathLike[str])) ->(str | None):
+def _template_basename(filename: str | os.PathLike[str]) -> str | None:
"""Given an input filename:
If the input looks like a template, then return the filename output should
be written to. Otherwise, return no result (None).
"""
- pass
+ basename = os.path.basename(filename)
+ if basename.lower().endswith('_t'):
+ return str(filename)[:-2]
+ elif basename.lower().endswith('.jinja'):
+ return str(filename)[:-6]
+ return None
-def copy_asset_file(source: (str | os.PathLike[str]), destination: (str |
- os.PathLike[str]), context: (dict[str, Any] | None)=None, renderer: (
- BaseRenderer | None)=None, *, force: bool=False) ->None:
+def copy_asset_file(source: str | os.PathLike[str], destination: str | os.PathLike[str],
+ context: dict[str, Any] | None = None,
+ renderer: BaseRenderer | None = None,
+ *,
+ force: bool = False) -> None:
"""Copy an asset file to destination.
On copying, it expands the template variables if context argument is given and
@@ -36,14 +50,50 @@ def copy_asset_file(source: (str | os.PathLike[str]), destination: (str |
:param renderer: The template engine. If not given, SphinxRenderer is used by default
:param bool force: Overwrite the destination file even if it exists.
"""
- pass
+ if not os.path.exists(source):
+ return
+
+ if os.path.isdir(destination):
+ # Use source filename if destination points a directory
+ destination = os.path.join(destination, os.path.basename(source))
+ else:
+ destination = str(destination)
+ if _template_basename(source) and context is not None:
+ if renderer is None:
+ from sphinx.util.template import SphinxRenderer
+ renderer = SphinxRenderer()
-def copy_asset(source: (str | os.PathLike[str]), destination: (str | os.
- PathLike[str]), excluded: PathMatcher=lambda path: False, context: (
- dict[str, Any] | None)=None, renderer: (BaseRenderer | None)=None,
- onerror: (Callable[[str, Exception], None] | None)=None, *, force: bool
- =False) ->None:
+ with open(source, encoding='utf-8') as fsrc:
+ template_content = fsrc.read()
+ rendered_template = renderer.render_string(template_content, context)
+
+ if (
+ not force
+ and os.path.exists(destination)
+ and template_content != rendered_template
+ ):
+ msg = __('Aborted attempted copy from rendered template %s to %s '
+ '(the destination path has existing data).')
+ logger.warning(msg, os.fsdecode(source), os.fsdecode(destination),
+ type='misc', subtype='copy_overwrite')
+ return
+
+ destination = _template_basename(destination) or destination
+ with open(destination, 'w', encoding='utf-8') as fdst:
+ msg = __('Writing evaluated template result to %s')
+ logger.info(msg, os.fsdecode(destination), type='misc',
+ subtype='template_evaluation')
+ fdst.write(rendered_template)
+ else:
+ copyfile(source, destination, force=force)
+
+
+def copy_asset(source: str | os.PathLike[str], destination: str | os.PathLike[str],
+ excluded: PathMatcher = lambda path: False,
+ context: dict[str, Any] | None = None, renderer: BaseRenderer | None = None,
+ onerror: Callable[[str, Exception], None] | None = None,
+ *, force: bool = False) -> None:
"""Copy asset files to destination recursively.
On copying, it expands the template variables if context argument is given and
@@ -59,4 +109,39 @@ def copy_asset(source: (str | os.PathLike[str]), destination: (str | os.
:param onerror: The error handler.
:param bool force: Overwrite the destination file even if it exists.
"""
- pass
+ if not os.path.exists(source):
+ return
+
+ if renderer is None:
+ from sphinx.util.template import SphinxRenderer
+ renderer = SphinxRenderer()
+
+ ensuredir(destination)
+ if os.path.isfile(source):
+ copy_asset_file(source, destination,
+ context=context,
+ renderer=renderer,
+ force=force)
+ return
+
+ for root, dirs, files in os.walk(source, followlinks=True):
+ reldir = relative_path(source, root)
+ for dir in dirs.copy():
+ if excluded(posixpath.join(reldir, dir)):
+ dirs.remove(dir)
+ else:
+ ensuredir(posixpath.join(destination, reldir, dir))
+
+ for filename in files:
+ if not excluded(posixpath.join(reldir, filename)):
+ try:
+ copy_asset_file(posixpath.join(root, filename),
+ posixpath.join(destination, reldir),
+ context=context,
+ renderer=renderer,
+ force=force)
+ except Exception as exc:
+ if onerror:
+ onerror(posixpath.join(root, filename), exc)
+ else:
+ raise
diff --git a/sphinx/util/http_date.py b/sphinx/util/http_date.py
index 0a1f2f186..4908101bc 100644
--- a/sphinx/util/http_date.py
+++ b/sphinx/util/http_date.py
@@ -2,21 +2,45 @@
Reference: https://www.rfc-editor.org/rfc/rfc7231#section-7.1.1.1
"""
+
import time
import warnings
from email.utils import parsedate_tz
+
from sphinx.deprecation import RemovedInSphinx90Warning
-_WEEKDAY_NAME = 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'
-_MONTH_NAME = ('', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug',
- 'Sep', 'Oct', 'Nov', 'Dec')
+
+_WEEKDAY_NAME = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
+_MONTH_NAME = ('', # Placeholder for indexing purposes
+ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+ 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
_GMT_OFFSET = float(time.localtime().tm_gmtoff)
-def epoch_to_rfc1123(epoch: float) ->str:
+def epoch_to_rfc1123(epoch: float) -> str:
"""Return HTTP-date string from epoch offset."""
- pass
+ yr, mn, dd, hh, mm, ss, wd, _yd, _tz = time.gmtime(epoch)
+ weekday_name = _WEEKDAY_NAME[wd]
+ month = _MONTH_NAME[mn]
+ return f'{weekday_name}, {dd:02} {month} {yr:04} {hh:02}:{mm:02}:{ss:02} GMT'
-def rfc1123_to_epoch(rfc1123: str) ->float:
+def rfc1123_to_epoch(rfc1123: str) -> float:
"""Return epoch offset from HTTP-date string."""
- pass
+ t = parsedate_tz(rfc1123)
+ if t is None:
+ raise ValueError
+ if not rfc1123.endswith(" GMT"):
+ warnings.warn(
+ "HTTP-date string does not meet RFC 7231 requirements "
+ f"(must end with 'GMT'): {rfc1123!r}",
+ RemovedInSphinx90Warning, stacklevel=3,
+ )
+ epoch_secs = time.mktime(time.struct_time(t[:9])) + _GMT_OFFSET
+ if (gmt_offset := t[9]) != 0:
+ warnings.warn(
+ "HTTP-date string does not meet RFC 7231 requirements "
+ f"(must be GMT time): {rfc1123!r}",
+ RemovedInSphinx90Warning, stacklevel=3,
+ )
+ return epoch_secs - (gmt_offset or 0)
+ return epoch_secs
diff --git a/sphinx/util/i18n.py b/sphinx/util/i18n.py
index c493eb2cf..2f2d500d9 100644
--- a/sphinx/util/i18n.py
+++ b/sphinx/util/i18n.py
@@ -1,47 +1,64 @@
"""Builder superclass for all builders."""
+
from __future__ import annotations
+
import os
import re
from datetime import datetime, timezone
from os import path
from typing import TYPE_CHECKING, NamedTuple
+
import babel.dates
from babel.messages.mofile import write_mo
from babel.messages.pofile import read_po
+
from sphinx.errors import SphinxError
from sphinx.locale import __
from sphinx.util import logging
-from sphinx.util.osutil import SEP, _last_modified_time, canon_path, relpath
+from sphinx.util.osutil import (
+ SEP,
+ _last_modified_time,
+ canon_path,
+ relpath,
+)
+
if TYPE_CHECKING:
import datetime as dt
from collections.abc import Iterator
from typing import Protocol, TypeAlias
+
from babel.core import Locale
- from sphinx.environment import BuildEnvironment
+ from sphinx.environment import BuildEnvironment
class DateFormatter(Protocol):
-
- def __call__(self, date: (dt.date | None)=..., format: str=...,
- locale: (str | Locale | None)=...) ->str:
- ...
-
+ def __call__( # NoQA: E704
+ self,
+ date: dt.date | None = ...,
+ format: str = ...,
+ locale: str | Locale | None = ...,
+ ) -> str: ...
class TimeFormatter(Protocol):
-
- def __call__(self, time: (dt.time | dt.datetime | float | None)=...,
- format: str=..., tzinfo: (dt.tzinfo | None)=..., locale: (str |
- Locale | None)=...) ->str:
- ...
-
+ def __call__( # NoQA: E704
+ self,
+ time: dt.time | dt.datetime | float | None = ...,
+ format: str = ...,
+ tzinfo: dt.tzinfo | None = ...,
+ locale: str | Locale | None = ...,
+ ) -> str: ...
class DatetimeFormatter(Protocol):
+ def __call__( # NoQA: E704
+ self,
+ datetime: dt.date | dt.time | float | None = ...,
+ format: str = ...,
+ tzinfo: dt.tzinfo | None = ...,
+ locale: str | Locale | None = ...,
+ ) -> str: ...
- def __call__(self, datetime: (dt.date | dt.time | float | None)=...,
- format: str=..., tzinfo: (dt.tzinfo | None)=..., locale: (str |
- Locale | None)=...) ->str:
- ...
Formatter: TypeAlias = DateFormatter | TimeFormatter | DatetimeFormatter
+
logger = logging.getLogger(__name__)
@@ -52,29 +69,221 @@ class LocaleFileInfoBase(NamedTuple):
class CatalogInfo(LocaleFileInfoBase):
- pass
+
+ @property
+ def po_file(self) -> str:
+ return self.domain + '.po'
+
+ @property
+ def mo_file(self) -> str:
+ return self.domain + '.mo'
+
+ @property
+ def po_path(self) -> str:
+ return path.join(self.base_dir, self.po_file)
+
+ @property
+ def mo_path(self) -> str:
+ return path.join(self.base_dir, self.mo_file)
+
+ def is_outdated(self) -> bool:
+ return (
+ not path.exists(self.mo_path) or
+ _last_modified_time(self.mo_path) < _last_modified_time(self.po_path))
+
+ def write_mo(self, locale: str, use_fuzzy: bool = False) -> None:
+ with open(self.po_path, encoding=self.charset) as file_po:
+ try:
+ po = read_po(file_po, locale)
+ except Exception as exc:
+ logger.warning(__('reading error: %s, %s'), self.po_path, exc)
+ return
+
+ with open(self.mo_path, 'wb') as file_mo:
+ try:
+ write_mo(file_mo, po, use_fuzzy)
+ except Exception as exc:
+ logger.warning(__('writing error: %s, %s'), self.mo_path, exc)
class CatalogRepository:
"""A repository for message catalogs."""
- def __init__(self, basedir: (str | os.PathLike[str]), locale_dirs: list
- [str], language: str, encoding: str) ->None:
+ def __init__(self, basedir: str | os.PathLike[str], locale_dirs: list[str],
+ language: str, encoding: str) -> None:
self.basedir = basedir
self._locale_dirs = locale_dirs
self.language = language
self.encoding = encoding
+ @property
+ def locale_dirs(self) -> Iterator[str]:
+ if not self.language:
+ return
-def docname_to_domain(docname: str, compaction: (bool | str)) ->str:
+ for locale_dir in self._locale_dirs:
+ locale_dir = path.join(self.basedir, locale_dir)
+ locale_path = path.join(locale_dir, self.language, 'LC_MESSAGES')
+ if path.exists(locale_path):
+ yield locale_dir
+ else:
+ logger.verbose(__('locale_dir %s does not exist'), locale_path)
+
+ @property
+ def pofiles(self) -> Iterator[tuple[str, str]]:
+ for locale_dir in self.locale_dirs:
+ basedir = path.join(locale_dir, self.language, 'LC_MESSAGES')
+ for root, dirnames, filenames in os.walk(basedir):
+ # skip dot-directories
+ for dirname in [d for d in dirnames if d.startswith('.')]:
+ dirnames.remove(dirname)
+
+ for filename in filenames:
+ if filename.endswith('.po'):
+ fullpath = path.join(root, filename)
+ yield basedir, relpath(fullpath, basedir)
+
+ @property
+ def catalogs(self) -> Iterator[CatalogInfo]:
+ for basedir, filename in self.pofiles:
+ domain = canon_path(path.splitext(filename)[0])
+ yield CatalogInfo(basedir, domain, self.encoding)
+
+
+def docname_to_domain(docname: str, compaction: bool | str) -> str:
"""Convert docname to domain for catalogs."""
- pass
+ if isinstance(compaction, str):
+ return compaction
+ if compaction:
+ return docname.split(SEP, 1)[0]
+ else:
+ return docname
-date_format_mappings = {'%a': 'EEE', '%A': 'EEEE', '%b': 'MMM', '%B':
- 'MMMM', '%c': 'medium', '%-d': 'd', '%d': 'dd', '%-H': 'H', '%H': 'HH',
- '%-I': 'h', '%I': 'hh', '%-j': 'D', '%j': 'DDD', '%-m': 'M', '%m': 'MM',
- '%-M': 'm', '%M': 'mm', '%p': 'a', '%-S': 's', '%S': 'ss', '%U': 'WW',
- '%w': 'e', '%-W': 'W', '%W': 'WW', '%x': 'medium', '%X': 'medium', '%y':
- 'YY', '%Y': 'yyyy', '%Z': 'zzz', '%z': 'ZZZ', '%%': '%'}
+# date_format mappings: ustrftime() to babel.dates.format_datetime()
+date_format_mappings = {
+ '%a': 'EEE', # Weekday as locale’s abbreviated name.
+ '%A': 'EEEE', # Weekday as locale’s full name.
+ '%b': 'MMM', # Month as locale’s abbreviated name.
+ '%B': 'MMMM', # Month as locale’s full name.
+ '%c': 'medium', # Locale’s appropriate date and time representation.
+ '%-d': 'd', # Day of the month as a decimal number.
+ '%d': 'dd', # Day of the month as a zero-padded decimal number.
+ '%-H': 'H', # Hour (24-hour clock) as a decimal number [0,23].
+ '%H': 'HH', # Hour (24-hour clock) as a zero-padded decimal number [00,23].
+ '%-I': 'h', # Hour (12-hour clock) as a decimal number [1,12].
+ '%I': 'hh', # Hour (12-hour clock) as a zero-padded decimal number [01,12].
+ '%-j': 'D', # Day of the year as a decimal number.
+ '%j': 'DDD', # Day of the year as a zero-padded decimal number.
+ '%-m': 'M', # Month as a decimal number.
+ '%m': 'MM', # Month as a zero-padded decimal number.
+ '%-M': 'm', # Minute as a decimal number [0,59].
+ '%M': 'mm', # Minute as a zero-padded decimal number [00,59].
+ '%p': 'a', # Locale’s equivalent of either AM or PM.
+ '%-S': 's', # Second as a decimal number.
+ '%S': 'ss', # Second as a zero-padded decimal number.
+ '%U': 'WW', # Week number of the year (Sunday as the first day of the week)
+ # as a zero padded decimal number. All days in a new year preceding
+ # the first Sunday are considered to be in week 0.
+ '%w': 'e', # Weekday as a decimal number, where 0 is Sunday and 6 is Saturday.
+ '%-W': 'W', # Week number of the year (Monday as the first day of the week)
+ # as a decimal number. All days in a new year preceding the first
+ # Monday are considered to be in week 0.
+ '%W': 'WW', # Week number of the year (Monday as the first day of the week)
+ # as a zero-padded decimal number.
+ '%x': 'medium', # Locale’s appropriate date representation.
+ '%X': 'medium', # Locale’s appropriate time representation.
+ '%y': 'YY', # Year without century as a zero-padded decimal number.
+ '%Y': 'yyyy', # Year with century as a decimal number.
+ '%Z': 'zzz', # Time zone name (no characters if no time zone exists).
+ '%z': 'ZZZ', # UTC offset in the form ±HHMM[SS[.ffffff]]
+ # (empty string if the object is naive).
+ '%%': '%',
+}
+
date_format_re = re.compile('(%s)' % '|'.join(date_format_mappings))
+
+
+def babel_format_date(date: datetime, format: str, locale: str,
+ formatter: Formatter = babel.dates.format_date) -> str:
+ # Check if we have the tzinfo attribute. If not we cannot do any time
+ # related formats.
+ if not hasattr(date, 'tzinfo'):
+ formatter = babel.dates.format_date
+
+ try:
+ return formatter(date, format, locale=locale)
+ except (ValueError, babel.core.UnknownLocaleError):
+ # fallback to English
+ return formatter(date, format, locale='en')
+ except AttributeError:
+ logger.warning(__('Invalid date format. Quote the string by single quote '
+ 'if you want to output it directly: %s'), format)
+ return format
+
+
+def format_date(
+ format: str, *, date: datetime | None = None, language: str,
+) -> str:
+ if date is None:
+ # If time is not specified, try to use $SOURCE_DATE_EPOCH variable
+ # See https://wiki.debian.org/ReproducibleBuilds/TimestampsProposal
+ source_date_epoch = os.getenv('SOURCE_DATE_EPOCH')
+ if source_date_epoch is not None:
+ date = datetime.fromtimestamp(float(source_date_epoch), tz=timezone.utc)
+ else:
+ date = datetime.now(tz=timezone.utc).astimezone()
+
+ result = []
+ tokens = date_format_re.split(format)
+ for token in tokens:
+ if token in date_format_mappings:
+ babel_format = date_format_mappings.get(token, '')
+
+ # Check if we have to use a different babel formatter then
+ # format_datetime, because we only want to format a date
+ # or a time.
+ function: Formatter
+ if token == '%x':
+ function = babel.dates.format_date
+ elif token == '%X':
+ function = babel.dates.format_time
+ else:
+ function = babel.dates.format_datetime
+
+ result.append(babel_format_date(date, babel_format, locale=language,
+ formatter=function))
+ else:
+ result.append(token)
+
+ return "".join(result)
+
+
+def get_image_filename_for_language(
+ filename: str | os.PathLike[str],
+ env: BuildEnvironment,
+) -> str:
+ root, ext = path.splitext(filename)
+ dirname = path.dirname(root)
+ docpath = path.dirname(env.docname)
+ try:
+ return env.config.figure_language_filename.format(
+ root=root,
+ ext=ext,
+ path=dirname and dirname + SEP,
+ basename=path.basename(root),
+ docpath=docpath and docpath + SEP,
+ language=env.config.language,
+ )
+ except KeyError as exc:
+ msg = f'Invalid figure_language_filename: {exc!r}'
+ raise SphinxError(msg) from exc
+
+
+def search_image_for_language(filename: str, env: BuildEnvironment) -> str:
+ translated = get_image_filename_for_language(filename, env)
+ _, abspath = env.relfn2path(translated)
+ if path.exists(abspath):
+ return translated
+ else:
+ return filename
diff --git a/sphinx/util/images.py b/sphinx/util/images.py
index 06f78c23a..70735c6cf 100644
--- a/sphinx/util/images.py
+++ b/sphinx/util/images.py
@@ -1,20 +1,32 @@
"""Image utility functions for Sphinx."""
+
from __future__ import annotations
+
import base64
from os import path
from typing import TYPE_CHECKING, NamedTuple, overload
+
import imagesize
+
if TYPE_CHECKING:
from os import PathLike
+
try:
from PIL import Image
PILLOW_AVAILABLE = True
except ImportError:
PILLOW_AVAILABLE = False
-mime_suffixes = {'.gif': 'image/gif', '.jpg': 'image/jpeg', '.png':
- 'image/png', '.pdf': 'application/pdf', '.svg': 'image/svg+xml',
- '.svgz': 'image/svg+xml', '.ai': 'application/illustrator', '.webp':
- 'image/webp'}
+
+mime_suffixes = {
+ '.gif': 'image/gif',
+ '.jpg': 'image/jpeg',
+ '.png': 'image/png',
+ '.pdf': 'application/pdf',
+ '.svg': 'image/svg+xml',
+ '.svgz': 'image/svg+xml',
+ '.ai': 'application/illustrator',
+ '.webp': 'image/webp',
+}
_suffix_from_mime = {v: k for k, v in reversed(mime_suffixes.items())}
@@ -22,3 +34,115 @@ class DataURI(NamedTuple):
mimetype: str
charset: str
data: bytes
+
+
+def get_image_size(filename: str) -> tuple[int, int] | None:
+ try:
+ size = imagesize.get(filename)
+ if size[0] == -1:
+ size = None
+ elif isinstance(size[0], float) or isinstance(size[1], float):
+ size = (int(size[0]), int(size[1]))
+
+ if size is None and PILLOW_AVAILABLE: # fallback to Pillow
+ with Image.open(filename) as im:
+ size = im.size
+
+ return size
+ except Exception:
+ return None
+
+
+@overload
+def guess_mimetype(filename: PathLike[str] | str, default: str) -> str:
+ ...
+
+
+@overload
+def guess_mimetype(filename: PathLike[str] | str, default: None = None) -> str | None:
+ ...
+
+
+def guess_mimetype(
+ filename: PathLike[str] | str = '',
+ default: str | None = None,
+) -> str | None:
+ ext = path.splitext(filename)[1].lower()
+ if ext in mime_suffixes:
+ return mime_suffixes[ext]
+ if path.exists(filename):
+ try:
+ imgtype = _image_type_from_file(filename)
+ except ValueError:
+ pass
+ else:
+ return 'image/' + imgtype
+ return default
+
+
+def get_image_extension(mimetype: str) -> str | None:
+ return _suffix_from_mime.get(mimetype)
+
+
+def parse_data_uri(uri: str) -> DataURI | None:
+ if not uri.startswith('data:'):
+ return None
+
+ # data:[<MIME-type>][;charset=<encoding>][;base64],<data>
+ mimetype = 'text/plain'
+ charset = 'US-ASCII'
+
+ properties, data = uri[5:].split(',', 1)
+ for prop in properties.split(';'):
+ if prop == 'base64':
+ pass # skip
+ elif prop.startswith('charset='):
+ charset = prop[8:]
+ elif prop:
+ mimetype = prop
+
+ image_data = base64.b64decode(data)
+ return DataURI(mimetype, charset, image_data)
+
+
+def _image_type_from_file(filename: PathLike[str] | str) -> str:
+ with open(filename, 'rb') as f:
+ header = f.read(32) # 32 bytes
+
+ # Bitmap
+ # https://en.wikipedia.org/wiki/BMP_file_format#Bitmap_file_header
+ if header.startswith(b'BM'):
+ return 'bmp'
+
+ # GIF
+ # https://en.wikipedia.org/wiki/GIF#File_format
+ if header.startswith((b'GIF87a', b'GIF89a')):
+ return 'gif'
+
+ # JPEG data
+ # https://en.wikipedia.org/wiki/JPEG_File_Interchange_Format#File_format_structure
+ if header.startswith(b'\xFF\xD8'):
+ return 'jpeg'
+
+ # Portable Network Graphics
+ # https://en.wikipedia.org/wiki/PNG#File_header
+ if header.startswith(b'\x89PNG\r\n\x1A\n'):
+ return 'png'
+
+ # Scalable Vector Graphics
+ # https://svgwg.org/svg2-draft/struct.html
+ if b'<svg' in header.lower():
+ return 'svg+xml'
+
+ # TIFF
+ # https://en.wikipedia.org/wiki/TIFF#Byte_order
+ if header.startswith((b'MM', b'II')):
+ return 'tiff'
+
+ # WebP
+ # https://en.wikipedia.org/wiki/WebP#Technology
+ if header.startswith(b'RIFF') and header[8:12] == b'WEBP':
+ return 'webp'
+
+ msg = 'Could not detect image type!'
+ raise ValueError(msg)
diff --git a/sphinx/util/index_entries.py b/sphinx/util/index_entries.py
index 56ebb9a85..100468429 100644
--- a/sphinx/util/index_entries.py
+++ b/sphinx/util/index_entries.py
@@ -1,6 +1,27 @@
from __future__ import annotations
-def _split_into(n: int, type: str, value: str) ->list[str]:
+def split_index_msg(entry_type: str, value: str) -> list[str]:
+ # new entry types must be listed in util/nodes.py!
+ if entry_type == 'single':
+ try:
+ return _split_into(2, 'single', value)
+ except ValueError:
+ return _split_into(1, 'single', value)
+ if entry_type == 'pair':
+ return _split_into(2, 'pair', value)
+ if entry_type == 'triple':
+ return _split_into(3, 'triple', value)
+ if entry_type in {'see', 'seealso'}:
+ return _split_into(2, 'see', value)
+ msg = f'invalid {entry_type} index entry {value!r}'
+ raise ValueError(msg)
+
+
+def _split_into(n: int, type: str, value: str) -> list[str]:
"""Split an index entry into a given number of parts at semicolons."""
- pass
+ parts = [x.strip() for x in value.split(';', n - 1)]
+ if len(list(filter(None, parts))) < n:
+ msg = f'invalid {type} index entry {value!r}'
+ raise ValueError(msg)
+ return parts
diff --git a/sphinx/util/inspect.py b/sphinx/util/inspect.py
index 5e7c1ae78..096da877c 100644
--- a/sphinx/util/inspect.py
+++ b/sphinx/util/inspect.py
@@ -1,5 +1,7 @@
"""Helpers for inspecting Python modules."""
+
from __future__ import annotations
+
import ast
import builtins
import contextlib
@@ -16,40 +18,51 @@ from inspect import Parameter, Signature
from io import StringIO
from types import ClassMethodDescriptorType, MethodDescriptorType, WrapperDescriptorType
from typing import TYPE_CHECKING, Any, ForwardRef
+
from sphinx.pycode.ast import unparse as ast_unparse
from sphinx.util import logging
from sphinx.util.typing import stringify_annotation
+
if TYPE_CHECKING:
from collections.abc import Callable, Sequence
from inspect import _ParameterKind
from types import MethodType, ModuleType
from typing import Final, Protocol, TypeAlias
- from typing_extensions import TypeIs
+ from typing_extensions import TypeIs
class _SupportsGet(Protocol):
-
- def __get__(self, __instance: Any, __owner: (type | None)=...) ->Any:
- ...
-
+ def __get__(self, __instance: Any, __owner: type | None = ...) -> Any: ... # NoQA: E704
class _SupportsSet(Protocol):
-
- def __set__(self, __instance: Any, __value: Any) ->None:
- ...
-
+ # instance and value are contravariants but we do not need that precision
+ def __set__(self, __instance: Any, __value: Any) -> None: ... # NoQA: E704
class _SupportsDelete(Protocol):
+ # instance is contravariant but we do not need that precision
+ def __delete__(self, __instance: Any) -> None: ... # NoQA: E704
+
+ _RoutineType: TypeAlias = (
+ types.FunctionType
+ | types.LambdaType
+ | types.MethodType
+ | types.BuiltinFunctionType
+ | types.BuiltinMethodType
+ | types.WrapperDescriptorType
+ | types.MethodDescriptorType
+ | types.ClassMethodDescriptorType
+ )
+ _SignatureType: TypeAlias = (
+ Callable[..., Any]
+ | staticmethod
+ | classmethod
+ )
- def __delete__(self, __instance: Any) ->None:
- ...
- _RoutineType: TypeAlias = (types.FunctionType | types.LambdaType |
- types.MethodType | types.BuiltinFunctionType | types.
- BuiltinMethodType | types.WrapperDescriptorType | types.
- MethodDescriptorType | types.ClassMethodDescriptorType)
- _SignatureType: TypeAlias = Callable[..., Any] | staticmethod | classmethod
logger = logging.getLogger(__name__)
-memory_address_re = re.compile(' at 0x[0-9a-f]{8,16}(?=>)', re.IGNORECASE)
+
+memory_address_re = re.compile(r' at 0x[0-9a-f]{8,16}(?=>)', re.IGNORECASE)
+
+# re-export as is
isasyncgenfunction = inspect.isasyncgenfunction
ismethod = inspect.ismethod
ismethoddescriptor = inspect.ismethoddescriptor
@@ -57,15 +70,23 @@ isclass = inspect.isclass
ismodule = inspect.ismodule
-def unwrap(obj: Any) ->Any:
+def unwrap(obj: Any) -> Any:
"""Get an original object from wrapped object (wrapped functions).
Mocked objects are returned as is.
"""
- pass
+ if hasattr(obj, '__sphinx_mock__'):
+ # Skip unwrapping mock object to avoid RecursionError
+ return obj
+
+ try:
+ return inspect.unwrap(obj)
+ except ValueError:
+ # might be a mock object
+ return obj
-def unwrap_all(obj: Any, *, stop: (Callable[[Any], bool] | None)=None) ->Any:
+def unwrap_all(obj: Any, *, stop: Callable[[Any], bool] | None = None) -> Any:
"""Get an original object from wrapped object.
Unlike :func:`unwrap`, this unwraps partial functions, wrapped functions,
@@ -74,194 +95,395 @@ def unwrap_all(obj: Any, *, stop: (Callable[[Any], bool] | None)=None) ->Any:
When specified, *stop* is a predicate indicating whether an object should
be unwrapped or not.
"""
- pass
+ if callable(stop):
+ while not stop(obj):
+ if ispartial(obj):
+ obj = obj.func
+ elif inspect.isroutine(obj) and hasattr(obj, '__wrapped__'):
+ obj = obj.__wrapped__
+ elif isclassmethod(obj) or isstaticmethod(obj):
+ obj = obj.__func__
+ else:
+ return obj
+ return obj # in case the while loop never starts
+
+ while True:
+ if ispartial(obj):
+ obj = obj.func
+ elif inspect.isroutine(obj) and hasattr(obj, '__wrapped__'):
+ obj = obj.__wrapped__
+ elif isclassmethod(obj) or isstaticmethod(obj):
+ obj = obj.__func__
+ else:
+ return obj
-def getall(obj: Any) ->(Sequence[str] | None):
+def getall(obj: Any) -> Sequence[str] | None:
"""Get the ``__all__`` attribute of an object as a sequence.
This returns ``None`` if the given ``obj.__all__`` does not exist and
raises :exc:`ValueError` if ``obj.__all__`` is not a list or tuple of
strings.
"""
- pass
+ __all__ = safe_getattr(obj, '__all__', None)
+ if __all__ is None:
+ return None
+ if isinstance(__all__, list | tuple) and all(isinstance(e, str) for e in __all__):
+ return __all__
+ raise ValueError(__all__)
-def getannotations(obj: Any) ->Mapping[str, Any]:
+def getannotations(obj: Any) -> Mapping[str, Any]:
"""Safely get the ``__annotations__`` attribute of an object."""
- pass
+ __annotations__ = safe_getattr(obj, '__annotations__', None)
+ if isinstance(__annotations__, Mapping):
+ return __annotations__
+ return {}
-def getglobals(obj: Any) ->Mapping[str, Any]:
+def getglobals(obj: Any) -> Mapping[str, Any]:
"""Safely get :attr:`obj.__globals__ <function.__globals__>`."""
- pass
+ __globals__ = safe_getattr(obj, '__globals__', None)
+ if isinstance(__globals__, Mapping):
+ return __globals__
+ return {}
-def getmro(obj: Any) ->tuple[type, ...]:
+def getmro(obj: Any) -> tuple[type, ...]:
"""Safely get :attr:`obj.__mro__ <class.__mro__>`."""
- pass
+ __mro__ = safe_getattr(obj, '__mro__', None)
+ if isinstance(__mro__, tuple):
+ return __mro__
+ return ()
-def getorigbases(obj: Any) ->(tuple[Any, ...] | None):
+def getorigbases(obj: Any) -> tuple[Any, ...] | None:
"""Safely get ``obj.__orig_bases__``.
This returns ``None`` if the object is not a class or if ``__orig_bases__``
is not well-defined (e.g., a non-tuple object or an empty sequence).
"""
- pass
+ if not isclass(obj):
+ return None
+ # Get __orig_bases__ from obj.__dict__ to avoid accessing the parent's __orig_bases__.
+ # refs: https://github.com/sphinx-doc/sphinx/issues/9607
+ __dict__ = safe_getattr(obj, '__dict__', {})
+ __orig_bases__ = __dict__.get('__orig_bases__')
+ if isinstance(__orig_bases__, tuple) and len(__orig_bases__) > 0:
+ return __orig_bases__
+ return None
-def getslots(obj: Any) ->(dict[str, Any] | dict[str, None] | None):
+
+def getslots(obj: Any) -> dict[str, Any] | dict[str, None] | None:
"""Safely get :term:`obj.__slots__ <__slots__>` as a dictionary if any.
- This returns ``None`` if ``obj.__slots__`` does not exist.
- This raises a :exc:`TypeError` if *obj* is not a class.
- This raises a :exc:`ValueError` if ``obj.__slots__`` is invalid.
"""
- pass
-
-
-def isenumclass(x: Any) ->TypeIs[type[enum.Enum]]:
+ if not isclass(obj):
+ raise TypeError
+
+ __slots__ = safe_getattr(obj, '__slots__', None)
+ if __slots__ is None:
+ return None
+ elif isinstance(__slots__, dict):
+ return __slots__
+ elif isinstance(__slots__, str):
+ return {__slots__: None}
+ elif isinstance(__slots__, list | tuple):
+ return dict.fromkeys(__slots__)
+ else:
+ raise ValueError
+
+
+def isenumclass(x: Any) -> TypeIs[type[enum.Enum]]:
"""Check if the object is an :class:`enumeration class <enum.Enum>`."""
- pass
+ return isclass(x) and issubclass(x, enum.Enum)
-def isenumattribute(x: Any) ->TypeIs[enum.Enum]:
+def isenumattribute(x: Any) -> TypeIs[enum.Enum]:
"""Check if the object is an enumeration attribute."""
- pass
+ return isinstance(x, enum.Enum)
-def unpartial(obj: Any) ->Any:
+def unpartial(obj: Any) -> Any:
"""Get an original object from a partial-like object.
If *obj* is not a partial object, it is returned as is.
.. seealso:: :func:`ispartial`
"""
- pass
+ while ispartial(obj):
+ obj = obj.func
+ return obj
-def ispartial(obj: Any) ->TypeIs[partial | partialmethod]:
+def ispartial(obj: Any) -> TypeIs[partial | partialmethod]:
"""Check if the object is a partial function or method."""
- pass
+ return isinstance(obj, partial | partialmethod)
-def isclassmethod(obj: Any, cls: Any=None, name: (str | None)=None) ->TypeIs[
- classmethod]:
+def isclassmethod(
+ obj: Any,
+ cls: Any = None,
+ name: str | None = None,
+) -> TypeIs[classmethod]:
"""Check if the object is a :class:`classmethod`."""
- pass
-
-
-def isstaticmethod(obj: Any, cls: Any=None, name: (str | None)=None) ->TypeIs[
- staticmethod]:
+ if isinstance(obj, classmethod):
+ return True
+ if ismethod(obj) and obj.__self__ is not None and isclass(obj.__self__):
+ return True
+ if cls and name:
+ # trace __mro__ if the method is defined in parent class
+ sentinel = object()
+ for basecls in getmro(cls):
+ meth = basecls.__dict__.get(name, sentinel)
+ if meth is not sentinel:
+ return isclassmethod(meth)
+ return False
+
+
+def isstaticmethod(
+ obj: Any,
+ cls: Any = None,
+ name: str | None = None,
+) -> TypeIs[staticmethod]:
"""Check if the object is a :class:`staticmethod`."""
- pass
-
-
-def isdescriptor(x: Any) ->TypeIs[_SupportsGet | _SupportsSet | _SupportsDelete
- ]:
+ if isinstance(obj, staticmethod):
+ return True
+ if cls and name:
+ # trace __mro__ if the method is defined in parent class
+ sentinel = object()
+ for basecls in getattr(cls, '__mro__', [cls]):
+ meth = basecls.__dict__.get(name, sentinel)
+ if meth is not sentinel:
+ return isinstance(meth, staticmethod)
+ return False
+
+
+def isdescriptor(x: Any) -> TypeIs[_SupportsGet | _SupportsSet | _SupportsDelete]:
"""Check if the object is a :external+python:term:`descriptor`."""
- pass
+ return any(
+ callable(safe_getattr(x, item, None)) for item in ('__get__', '__set__', '__delete__')
+ )
-def isabstractmethod(obj: Any) ->bool:
+def isabstractmethod(obj: Any) -> bool:
"""Check if the object is an :func:`abstractmethod`."""
- pass
+ return safe_getattr(obj, '__isabstractmethod__', False) is True
-def isboundmethod(method: MethodType) ->bool:
+def isboundmethod(method: MethodType) -> bool:
"""Check if the method is a bound method."""
- pass
+ return safe_getattr(method, '__self__', None) is not None
-def is_cython_function_or_method(obj: Any) ->bool:
+def is_cython_function_or_method(obj: Any) -> bool:
"""Check if the object is a function or method in cython."""
- pass
+ try:
+ return obj.__class__.__name__ == 'cython_function_or_method'
+ except AttributeError:
+ return False
-_DESCRIPTOR_LIKE: Final[tuple[type, ...]] = (ClassMethodDescriptorType,
- MethodDescriptorType, WrapperDescriptorType)
+_DESCRIPTOR_LIKE: Final[tuple[type, ...]] = (
+ ClassMethodDescriptorType,
+ MethodDescriptorType,
+ WrapperDescriptorType,
+)
-def isattributedescriptor(obj: Any) ->bool:
+def isattributedescriptor(obj: Any) -> bool:
"""Check if the object is an attribute-like descriptor."""
- pass
-
-
-def is_singledispatch_function(obj: Any) ->bool:
+ if inspect.isdatadescriptor(obj):
+ # data descriptor is kind of attribute
+ return True
+ if isdescriptor(obj):
+ # non data descriptor
+ unwrapped = unwrap(obj)
+ if isfunction(unwrapped) or isbuiltin(unwrapped) or ismethod(unwrapped):
+ # attribute must not be either function, builtin and method
+ return False
+ if is_cython_function_or_method(unwrapped):
+ # attribute must not be either function and method (for cython)
+ return False
+ if isclass(unwrapped):
+ # attribute must not be a class
+ return False
+ if isinstance(unwrapped, _DESCRIPTOR_LIKE):
+ # attribute must not be a method descriptor
+ return False
+ # attribute must not be an instancemethod (C-API)
+ return type(unwrapped).__name__ != 'instancemethod'
+ return False
+
+
+def is_singledispatch_function(obj: Any) -> bool:
"""Check if the object is a :func:`~functools.singledispatch` function."""
- pass
+ return (
+ inspect.isfunction(obj)
+ and hasattr(obj, 'dispatch')
+ and hasattr(obj, 'register')
+ and obj.dispatch.__module__ == 'functools'
+ )
-def is_singledispatch_method(obj: Any) ->TypeIs[singledispatchmethod]:
+def is_singledispatch_method(obj: Any) -> TypeIs[singledispatchmethod]:
"""Check if the object is a :class:`~functools.singledispatchmethod`."""
- pass
+ return isinstance(obj, singledispatchmethod)
-def isfunction(obj: Any) ->TypeIs[types.FunctionType]:
+def isfunction(obj: Any) -> TypeIs[types.FunctionType]:
"""Check if the object is a user-defined function.
Partial objects are unwrapped before checking them.
.. seealso:: :external+python:func:`inspect.isfunction`
"""
- pass
+ return inspect.isfunction(unpartial(obj))
-def isbuiltin(obj: Any) ->TypeIs[types.BuiltinFunctionType]:
+def isbuiltin(obj: Any) -> TypeIs[types.BuiltinFunctionType]:
"""Check if the object is a built-in function or method.
Partial objects are unwrapped before checking them.
.. seealso:: :external+python:func:`inspect.isbuiltin`
"""
- pass
+ return inspect.isbuiltin(unpartial(obj))
-def isroutine(obj: Any) ->TypeIs[_RoutineType]:
+def isroutine(obj: Any) -> TypeIs[_RoutineType]:
"""Check if the object is a kind of function or method.
Partial objects are unwrapped before checking them.
.. seealso:: :external+python:func:`inspect.isroutine`
"""
- pass
+ return inspect.isroutine(unpartial(obj))
-def iscoroutinefunction(obj: Any) ->TypeIs[Callable[..., types.CoroutineType]]:
+def iscoroutinefunction(obj: Any) -> TypeIs[Callable[..., types.CoroutineType]]:
"""Check if the object is a :external+python:term:`coroutine` function."""
- pass
+ obj = unwrap_all(obj, stop=_is_wrapped_coroutine)
+ return inspect.iscoroutinefunction(obj)
-def _is_wrapped_coroutine(obj: Any) ->bool:
+def _is_wrapped_coroutine(obj: Any) -> bool:
"""Check if the object is wrapped coroutine-function."""
- pass
+ if isstaticmethod(obj) or isclassmethod(obj) or ispartial(obj):
+ # staticmethod, classmethod and partial method are not a wrapped coroutine-function
+ # Note: Since 3.10, staticmethod and classmethod becomes a kind of wrappers
+ return False
+ return hasattr(obj, '__wrapped__')
-def isproperty(obj: Any) ->TypeIs[property | cached_property]:
+def isproperty(obj: Any) -> TypeIs[property | cached_property]:
"""Check if the object is property (possibly cached)."""
- pass
+ return isinstance(obj, property | cached_property)
-def isgenericalias(obj: Any) ->TypeIs[types.GenericAlias]:
+def isgenericalias(obj: Any) -> TypeIs[types.GenericAlias]:
"""Check if the object is a generic alias."""
- pass
+ return isinstance(obj, types.GenericAlias | typing._BaseGenericAlias) # type: ignore[attr-defined]
-def safe_getattr(obj: Any, name: str, *defargs: Any) ->Any:
+def safe_getattr(obj: Any, name: str, *defargs: Any) -> Any:
"""A getattr() that turns all exceptions into AttributeErrors."""
- pass
-
-
-def object_description(obj: Any, *, _seen: frozenset[int]=frozenset()) ->str:
+ try:
+ return getattr(obj, name, *defargs)
+ except Exception as exc:
+ # sometimes accessing a property raises an exception (e.g.
+ # NotImplementedError), so let's try to read the attribute directly
+ try:
+ # In case the object does weird things with attribute access
+ # such that accessing `obj.__dict__` may raise an exception
+ return obj.__dict__[name]
+ except Exception:
+ pass
+
+ # this is a catch-all for all the weird things that some modules do
+ # with attribute access
+ if defargs:
+ return defargs[0]
+
+ raise AttributeError(name) from exc
+
+
+def object_description(obj: Any, *, _seen: frozenset[int] = frozenset()) -> str:
"""A repr() implementation that returns text safe to use in reST context.
Maintains a set of 'seen' object IDs to detect and avoid infinite recursion.
"""
- pass
-
-
-def is_builtin_class_method(obj: Any, attr_name: str) ->bool:
+ seen = _seen
+ if isinstance(obj, dict):
+ if id(obj) in seen:
+ return 'dict(...)'
+ seen |= {id(obj)}
+ try:
+ sorted_keys = sorted(obj)
+ except TypeError:
+ # Cannot sort dict keys, fall back to using descriptions as a sort key
+ sorted_keys = sorted(obj, key=lambda k: object_description(k, _seen=seen))
+
+ items = (
+ (object_description(key, _seen=seen), object_description(obj[key], _seen=seen))
+ for key in sorted_keys
+ )
+ return '{%s}' % ', '.join(f'{key}: {value}' for (key, value) in items)
+ elif isinstance(obj, set):
+ if id(obj) in seen:
+ return 'set(...)'
+ seen |= {id(obj)}
+ try:
+ sorted_values = sorted(obj)
+ except TypeError:
+ # Cannot sort set values, fall back to using descriptions as a sort key
+ sorted_values = sorted(obj, key=lambda x: object_description(x, _seen=seen))
+ return '{%s}' % ', '.join(object_description(x, _seen=seen) for x in sorted_values)
+ elif isinstance(obj, frozenset):
+ if id(obj) in seen:
+ return 'frozenset(...)'
+ seen |= {id(obj)}
+ try:
+ sorted_values = sorted(obj)
+ except TypeError:
+ # Cannot sort frozenset values, fall back to using descriptions as a sort key
+ sorted_values = sorted(obj, key=lambda x: object_description(x, _seen=seen))
+ return 'frozenset({%s})' % ', '.join(
+ object_description(x, _seen=seen) for x in sorted_values
+ )
+ elif isinstance(obj, enum.Enum):
+ if obj.__repr__.__func__ is not enum.Enum.__repr__: # type: ignore[attr-defined]
+ return repr(obj)
+ return f'{obj.__class__.__name__}.{obj.name}'
+ elif isinstance(obj, tuple):
+ if id(obj) in seen:
+ return 'tuple(...)'
+ seen |= frozenset([id(obj)])
+ return '({}{})'.format(
+ ', '.join(object_description(x, _seen=seen) for x in obj),
+ ',' * (len(obj) == 1),
+ )
+ elif isinstance(obj, list):
+ if id(obj) in seen:
+ return 'list(...)'
+ seen |= {id(obj)}
+ return '[%s]' % ', '.join(object_description(x, _seen=seen) for x in obj)
+
+ try:
+ s = repr(obj)
+ except Exception as exc:
+ raise ValueError from exc
+ # Strip non-deterministic memory addresses such as
+ # ``<__main__.A at 0x7f68cb685710>``
+ s = memory_address_re.sub('', s)
+ return s.replace('\n', ' ')
+
+
+def is_builtin_class_method(obj: Any, attr_name: str) -> bool:
"""Check whether *attr_name* is implemented on a builtin class.
>>> is_builtin_class_method(int, '__init__')
@@ -271,19 +493,31 @@ def is_builtin_class_method(obj: Any, attr_name: str) ->bool:
This function is needed since CPython implements ``int.__init__`` via
descriptors, but PyPy implementation is written in pure Python code.
"""
- pass
+ mro = getmro(obj)
+
+ try:
+ cls = next(c for c in mro if attr_name in safe_getattr(c, '__dict__', {}))
+ except StopIteration:
+ return False
+
+ try:
+ name = safe_getattr(cls, '__name__')
+ except AttributeError:
+ return False
+
+ return getattr(builtins, name, None) is cls
class DefaultValue:
"""A simple wrapper for default value of the parameters of overload functions."""
- def __init__(self, value: str) ->None:
+ def __init__(self, value: str) -> None:
self.value = value
- def __eq__(self, other: object) ->bool:
+ def __eq__(self, other: object) -> bool:
return self.value == other
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return self.value
@@ -293,46 +527,53 @@ class TypeAliasForwardRef:
This avoids the error on evaluating the type inside :func:`typing.get_type_hints()`.
"""
- def __init__(self, name: str) ->None:
+ def __init__(self, name: str) -> None:
self.name = name
- def __call__(self) ->None:
+ def __call__(self) -> None:
+ # Dummy method to imitate special typing classes
pass
- def __eq__(self, other: Any) ->bool:
+ def __eq__(self, other: Any) -> bool:
return self.name == other
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash(self.name)
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return self.name
class TypeAliasModule:
"""Pseudo module class for :confval:`autodoc_type_aliases`."""
- def __init__(self, modname: str, mapping: Mapping[str, str]) ->None:
+ def __init__(self, modname: str, mapping: Mapping[str, str]) -> None:
self.__modname = modname
self.__mapping = mapping
+
self.__module: ModuleType | None = None
- def __getattr__(self, name: str) ->Any:
+ def __getattr__(self, name: str) -> Any:
fullname = '.'.join(filter(None, [self.__modname, name]))
if fullname in self.__mapping:
+ # exactly matched
return TypeAliasForwardRef(self.__mapping[fullname])
else:
prefix = fullname + '.'
- nested = {k: v for k, v in self.__mapping.items() if k.
- startswith(prefix)}
+ nested = {k: v for k, v in self.__mapping.items() if k.startswith(prefix)}
if nested:
+ # sub modules or classes found
return TypeAliasModule(fullname, nested)
else:
+ # no sub modules or classes found.
try:
+ # return the real submodule if exists
return import_module(fullname)
except ImportError:
+ # return the real class
if self.__module is None:
self.__module = import_module(self.__modname)
+
return getattr(self.__module, name)
@@ -342,58 +583,166 @@ class TypeAliasNamespace(dict[str, Any]):
Useful for looking up nested objects via ``namespace.foo.bar.Class``.
"""
- def __init__(self, mapping: Mapping[str, str]) ->None:
+ def __init__(self, mapping: Mapping[str, str]) -> None:
super().__init__()
self.__mapping = mapping
- def __getitem__(self, key: str) ->Any:
+ def __getitem__(self, key: str) -> Any:
if key in self.__mapping:
+ # exactly matched
return TypeAliasForwardRef(self.__mapping[key])
else:
prefix = key + '.'
- nested = {k: v for k, v in self.__mapping.items() if k.
- startswith(prefix)}
+ nested = {k: v for k, v in self.__mapping.items() if k.startswith(prefix)}
if nested:
+ # sub modules or classes found
return TypeAliasModule(key, nested)
else:
raise KeyError
-def _should_unwrap(subject: _SignatureType) ->bool:
+def _should_unwrap(subject: _SignatureType) -> bool:
"""Check the function should be unwrapped on getting signature."""
- pass
-
-
-def signature(subject: _SignatureType, bound_method: bool=False,
- type_aliases: (Mapping[str, str] | None)=None) ->Signature:
+ __globals__ = getglobals(subject)
+ # contextmanger should be unwrapped
+ return (
+ __globals__.get('__name__') == 'contextlib'
+ and __globals__.get('__file__') == contextlib.__file__
+ )
+
+
+def signature(
+ subject: _SignatureType,
+ bound_method: bool = False,
+ type_aliases: Mapping[str, str] | None = None,
+) -> Signature:
"""Return a Signature object for the given *subject*.
:param bound_method: Specify *subject* is a bound method or not
"""
- pass
+ if type_aliases is None:
+ type_aliases = {}
+ try:
+ if _should_unwrap(subject):
+ signature = inspect.signature(subject) # type: ignore[arg-type]
+ else:
+ signature = inspect.signature(subject, follow_wrapped=True) # type: ignore[arg-type]
+ except ValueError:
+ # follow built-in wrappers up (ex. functools.lru_cache)
+ signature = inspect.signature(subject) # type: ignore[arg-type]
+ parameters = list(signature.parameters.values())
+ return_annotation = signature.return_annotation
+
+ try:
+ # Resolve annotations using ``get_type_hints()`` and type_aliases.
+ localns = TypeAliasNamespace(type_aliases)
+ annotations = typing.get_type_hints(subject, None, localns, include_extras=True)
+ for i, param in enumerate(parameters):
+ if param.name in annotations:
+ annotation = annotations[param.name]
+ if isinstance(annotation, TypeAliasForwardRef):
+ annotation = annotation.name
+ parameters[i] = param.replace(annotation=annotation)
+ if 'return' in annotations:
+ if isinstance(annotations['return'], TypeAliasForwardRef):
+ return_annotation = annotations['return'].name
+ else:
+ return_annotation = annotations['return']
+ except Exception:
+ # ``get_type_hints()`` does not support some kind of objects like partial,
+ # ForwardRef and so on.
+ pass
-def evaluate_signature(sig: Signature, globalns: (dict[str, Any] | None)=
- None, localns: (dict[str, Any] | None)=None) ->Signature:
+ if bound_method:
+ if inspect.ismethod(subject):
+ # ``inspect.signature()`` considers the subject is a bound method and removes
+ # first argument from signature. Therefore no skips are needed here.
+ pass
+ else:
+ if len(parameters) > 0:
+ parameters.pop(0)
+
+ # To allow to create signature object correctly for pure python functions,
+ # pass an internal parameter __validate_parameters__=False to Signature
+ #
+ # For example, this helps a function having a default value `inspect._empty`.
+ # refs: https://github.com/sphinx-doc/sphinx/issues/7935
+ return Signature(
+ parameters, return_annotation=return_annotation, __validate_parameters__=False
+ )
+
+
+def evaluate_signature(
+ sig: Signature,
+ globalns: dict[str, Any] | None = None,
+ localns: dict[str, Any] | None = None,
+) -> Signature:
"""Evaluate unresolved type annotations in a signature object."""
- pass
+ if globalns is None:
+ globalns = {}
+ if localns is None:
+ localns = globalns
+ parameters = list(sig.parameters.values())
+ for i, param in enumerate(parameters):
+ if param.annotation:
+ annotation = _evaluate(param.annotation, globalns, localns)
+ parameters[i] = param.replace(annotation=annotation)
-def _evaluate_forwardref(ref: ForwardRef, globalns: (dict[str, Any] | None),
- localns: (dict[str, Any] | None)) ->Any:
- """Evaluate a forward reference."""
- pass
+ return_annotation = sig.return_annotation
+ if return_annotation:
+ return_annotation = _evaluate(return_annotation, globalns, localns)
+
+ return sig.replace(parameters=parameters, return_annotation=return_annotation)
-def _evaluate(annotation: Any, globalns: dict[str, Any], localns: dict[str,
- Any]) ->Any:
+def _evaluate_forwardref(
+ ref: ForwardRef,
+ globalns: dict[str, Any] | None,
+ localns: dict[str, Any] | None,
+) -> Any:
+ """Evaluate a forward reference."""
+ if sys.version_info >= (3, 12, 4):
+ # ``type_params`` were added in 3.13 and the signature of _evaluate()
+ # is not backward-compatible (it was backported to 3.12.4, so anything
+ # before 3.12.4 still has the old signature).
+ #
+ # See: https://github.com/python/cpython/pull/118104.
+ return ref._evaluate(globalns, localns, {}, recursive_guard=frozenset()) # type: ignore[arg-type, misc]
+ return ref._evaluate(globalns, localns, frozenset())
+
+
+def _evaluate(
+ annotation: Any,
+ globalns: dict[str, Any],
+ localns: dict[str, Any],
+) -> Any:
"""Evaluate unresolved type annotation."""
- pass
+ try:
+ if isinstance(annotation, str):
+ ref = ForwardRef(annotation, True)
+ annotation = _evaluate_forwardref(ref, globalns, localns)
+
+ if isinstance(annotation, ForwardRef):
+ annotation = _evaluate_forwardref(ref, globalns, localns)
+ elif isinstance(annotation, str):
+ # might be a ForwardRef'ed annotation in overloaded functions
+ ref = ForwardRef(annotation, True)
+ annotation = _evaluate_forwardref(ref, globalns, localns)
+ except (NameError, TypeError):
+ # failed to evaluate type. skipped.
+ pass
+ return annotation
-def stringify_signature(sig: Signature, show_annotation: bool=True,
- show_return_annotation: bool=True, unqualified_typehints: bool=False
- ) ->str:
+
+def stringify_signature(
+ sig: Signature,
+ show_annotation: bool = True,
+ show_return_annotation: bool = True,
+ unqualified_typehints: bool = False,
+) -> str:
"""Stringify a :class:`~inspect.Signature` object.
:param show_annotation: If enabled, show annotations on the signature
@@ -401,21 +750,137 @@ def stringify_signature(sig: Signature, show_annotation: bool=True,
:param unqualified_typehints: If enabled, show annotations as unqualified
(ex. io.StringIO -> StringIO)
"""
- pass
+ if unqualified_typehints:
+ mode = 'smart'
+ else:
+ mode = 'fully-qualified'
+
+ EMPTY = Parameter.empty
+
+ args = []
+ last_kind = None
+ for param in sig.parameters.values():
+ if param.kind != Parameter.POSITIONAL_ONLY and last_kind == Parameter.POSITIONAL_ONLY:
+ # PEP-570: Separator for Positional Only Parameter: /
+ args.append('/')
+ if param.kind == Parameter.KEYWORD_ONLY and last_kind in (
+ Parameter.POSITIONAL_OR_KEYWORD,
+ Parameter.POSITIONAL_ONLY,
+ None,
+ ):
+ # PEP-3102: Separator for Keyword Only Parameter: *
+ args.append('*')
+
+ arg = StringIO()
+ if param.kind is Parameter.VAR_POSITIONAL:
+ arg.write('*' + param.name)
+ elif param.kind is Parameter.VAR_KEYWORD:
+ arg.write('**' + param.name)
+ else:
+ arg.write(param.name)
+
+ if show_annotation and param.annotation is not EMPTY:
+ arg.write(': ')
+ arg.write(stringify_annotation(param.annotation, mode)) # type: ignore[arg-type]
+ if param.default is not EMPTY:
+ if show_annotation and param.annotation is not EMPTY:
+ arg.write(' = ')
+ else:
+ arg.write('=')
+ arg.write(object_description(param.default))
+ args.append(arg.getvalue())
+ last_kind = param.kind
-def signature_from_str(signature: str) ->Signature:
- """Create a :class:`~inspect.Signature` object from a string."""
- pass
+ if last_kind is Parameter.POSITIONAL_ONLY:
+ # PEP-570: Separator for Positional Only Parameter: /
+ args.append('/')
+ concatenated_args = ', '.join(args)
+ if sig.return_annotation is EMPTY or not show_annotation or not show_return_annotation:
+ return f'({concatenated_args})'
+ else:
+ retann = stringify_annotation(sig.return_annotation, mode) # type: ignore[arg-type]
+ return f'({concatenated_args}) -> {retann}'
-def signature_from_ast(node: ast.FunctionDef, code: str='') ->Signature:
- """Create a :class:`~inspect.Signature` object from an AST node."""
- pass
+
+def signature_from_str(signature: str) -> Signature:
+ """Create a :class:`~inspect.Signature` object from a string."""
+ code = 'def func' + signature + ': pass'
+ module = ast.parse(code)
+ function = typing.cast(ast.FunctionDef, module.body[0])
+
+ return signature_from_ast(function, code)
-def getdoc(obj: Any, attrgetter: Callable=safe_getattr, allow_inherited:
- bool=False, cls: Any=None, name: (str | None)=None) ->(str | None):
+def signature_from_ast(node: ast.FunctionDef, code: str = '') -> Signature:
+ """Create a :class:`~inspect.Signature` object from an AST node."""
+ EMPTY = Parameter.empty
+
+ args: ast.arguments = node.args
+ defaults: tuple[ast.expr | None, ...] = tuple(args.defaults)
+ pos_only_offset = len(args.posonlyargs)
+ defaults_offset = pos_only_offset + len(args.args) - len(defaults)
+ # The sequence ``D = args.defaults`` contains non-None AST expressions,
+ # so we can use ``None`` as a sentinel value for that to indicate that
+ # there is no default value for a specific parameter.
+ #
+ # Let *p* be the number of positional-only and positional-or-keyword
+ # arguments. Note that ``0 <= len(D) <= p`` and ``D[0]`` is the default
+ # value corresponding to a positional-only *or* a positional-or-keyword
+ # argument. Since a non-default argument cannot follow a default argument,
+ # the sequence *D* can be completed on the left by adding None sentinels
+ # so that ``len(D) == p`` and ``D[i]`` is the *i*-th default argument.
+ defaults = (None,) * defaults_offset + defaults
+
+ # construct the parameter list
+ params: list[Parameter] = []
+
+ # positional-only arguments (introduced in Python 3.8)
+ for arg, defexpr in zip(args.posonlyargs, defaults, strict=False):
+ params.append(_define(Parameter.POSITIONAL_ONLY, arg, code, defexpr=defexpr))
+
+ # normal arguments
+ for arg, defexpr in zip(args.args, defaults[pos_only_offset:], strict=False):
+ params.append(_define(Parameter.POSITIONAL_OR_KEYWORD, arg, code, defexpr=defexpr))
+
+ # variadic positional argument (no possible default expression)
+ if args.vararg:
+ params.append(_define(Parameter.VAR_POSITIONAL, args.vararg, code, defexpr=None))
+
+ # keyword-only arguments
+ for arg, defexpr in zip(args.kwonlyargs, args.kw_defaults, strict=False):
+ params.append(_define(Parameter.KEYWORD_ONLY, arg, code, defexpr=defexpr))
+
+ # variadic keyword argument (no possible default expression)
+ if args.kwarg:
+ params.append(_define(Parameter.VAR_KEYWORD, args.kwarg, code, defexpr=None))
+
+ return_annotation = ast_unparse(node.returns, code) or EMPTY
+ return Signature(params, return_annotation=return_annotation)
+
+
+def _define(
+ kind: _ParameterKind,
+ arg: ast.arg,
+ code: str,
+ *,
+ defexpr: ast.expr | None,
+) -> Parameter:
+ EMPTY = Parameter.empty
+
+ default = EMPTY if defexpr is None else DefaultValue(ast_unparse(defexpr, code))
+ annotation = ast_unparse(arg.annotation, code) or EMPTY
+ return Parameter(arg.arg, kind, default=default, annotation=annotation)
+
+
+def getdoc(
+ obj: Any,
+ attrgetter: Callable = safe_getattr,
+ allow_inherited: bool = False,
+ cls: Any = None,
+ name: str | None = None,
+) -> str | None:
"""Get the docstring for the object.
This tries to obtain the docstring for some kind of objects additionally:
@@ -424,4 +889,46 @@ def getdoc(obj: Any, attrgetter: Callable=safe_getattr, allow_inherited:
* inherited docstring
* inherited decorated methods
"""
- pass
+ if cls and name and isclassmethod(obj, cls, name):
+ for basecls in getmro(cls):
+ meth = basecls.__dict__.get(name)
+ if meth and hasattr(meth, '__func__'):
+ doc: str | None = getdoc(meth.__func__)
+ if doc is not None or not allow_inherited:
+ return doc
+
+ doc = _getdoc_internal(obj)
+ if ispartial(obj) and doc == obj.__class__.__doc__:
+ return getdoc(obj.func)
+ elif doc is None and allow_inherited:
+ if cls and name:
+ # Check a docstring of the attribute or method from super classes.
+ for basecls in getmro(cls):
+ meth = safe_getattr(basecls, name, None)
+ if meth is not None:
+ doc = _getdoc_internal(meth)
+ if doc is not None:
+ break
+
+ if doc is None:
+ # retry using `inspect.getdoc()`
+ for basecls in getmro(cls):
+ meth = safe_getattr(basecls, name, None)
+ if meth is not None:
+ doc = inspect.getdoc(meth)
+ if doc is not None:
+ break
+
+ if doc is None:
+ doc = inspect.getdoc(obj)
+
+ return doc
+
+
+def _getdoc_internal(
+ obj: Any, attrgetter: Callable[[Any, str, Any], Any] = safe_getattr
+) -> str | None:
+ doc = attrgetter(obj, '__doc__', None)
+ if isinstance(doc, str):
+ return doc
+ return None
diff --git a/sphinx/util/inventory.py b/sphinx/util/inventory.py
index 5648e43b1..c48922cfb 100644
--- a/sphinx/util/inventory.py
+++ b/sphinx/util/inventory.py
@@ -1,15 +1,20 @@
"""Inventory utility functions for Sphinx."""
from __future__ import annotations
+
import os
import re
import zlib
from typing import IO, TYPE_CHECKING
+
from sphinx.locale import __
from sphinx.util import logging
+
BUFSIZE = 16 * 1024
logger = logging.getLogger(__name__)
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
+
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
from sphinx.util.typing import Inventory, InventoryItem
@@ -21,11 +26,186 @@ class InventoryFileReader:
This reader supports mixture of texts and compressed texts.
"""
- def __init__(self, stream: IO[bytes]) ->None:
+ def __init__(self, stream: IO[bytes]) -> None:
self.stream = stream
self.buffer = b''
self.eof = False
+ def read_buffer(self) -> None:
+ chunk = self.stream.read(BUFSIZE)
+ if chunk == b'':
+ self.eof = True
+ self.buffer += chunk
+
+ def readline(self) -> str:
+ pos = self.buffer.find(b'\n')
+ if pos != -1:
+ line = self.buffer[:pos].decode()
+ self.buffer = self.buffer[pos + 1:]
+ elif self.eof:
+ line = self.buffer.decode()
+ self.buffer = b''
+ else:
+ self.read_buffer()
+ line = self.readline()
+
+ return line
+
+ def readlines(self) -> Iterator[str]:
+ while not self.eof:
+ line = self.readline()
+ if line:
+ yield line
+
+ def read_compressed_chunks(self) -> Iterator[bytes]:
+ decompressor = zlib.decompressobj()
+ while not self.eof:
+ self.read_buffer()
+ yield decompressor.decompress(self.buffer)
+ self.buffer = b''
+ yield decompressor.flush()
+
+ def read_compressed_lines(self) -> Iterator[str]:
+ buf = b''
+ for chunk in self.read_compressed_chunks():
+ buf += chunk
+ pos = buf.find(b'\n')
+ while pos != -1:
+ yield buf[:pos].decode()
+ buf = buf[pos + 1:]
+ pos = buf.find(b'\n')
+
class InventoryFile:
- pass
+ @classmethod
+ def load(
+ cls: type[InventoryFile],
+ stream: IO[bytes],
+ uri: str,
+ joinfunc: Callable[[str, str], str],
+ ) -> Inventory:
+ reader = InventoryFileReader(stream)
+ line = reader.readline().rstrip()
+ if line == '# Sphinx inventory version 1':
+ return cls.load_v1(reader, uri, joinfunc)
+ elif line == '# Sphinx inventory version 2':
+ return cls.load_v2(reader, uri, joinfunc)
+ else:
+ raise ValueError('invalid inventory header: %s' % line)
+
+ @classmethod
+ def load_v1(
+ cls: type[InventoryFile],
+ stream: InventoryFileReader,
+ uri: str,
+ join: Callable[[str, str], str],
+ ) -> Inventory:
+ invdata: Inventory = {}
+ projname = stream.readline().rstrip()[11:]
+ version = stream.readline().rstrip()[11:]
+ for line in stream.readlines():
+ name, type, location = line.rstrip().split(None, 2)
+ location = join(uri, location)
+ # version 1 did not add anchors to the location
+ if type == 'mod':
+ type = 'py:module'
+ location += '#module-' + name
+ else:
+ type = 'py:' + type
+ location += '#' + name
+ invdata.setdefault(type, {})[name] = (projname, version, location, '-')
+ return invdata
+
+ @classmethod
+ def load_v2(
+ cls: type[InventoryFile],
+ stream: InventoryFileReader,
+ uri: str,
+ join: Callable[[str, str], str],
+ ) -> Inventory:
+ invdata: Inventory = {}
+ projname = stream.readline().rstrip()[11:]
+ version = stream.readline().rstrip()[11:]
+ # definition -> priority, location, display name
+ potential_ambiguities: dict[str, tuple[str, str, str]] = {}
+ actual_ambiguities = set()
+ line = stream.readline()
+ if 'zlib' not in line:
+ raise ValueError('invalid inventory header (not compressed): %s' % line)
+
+ for line in stream.read_compressed_lines():
+ # be careful to handle names with embedded spaces correctly
+ m = re.match(r'(.+?)\s+(\S+)\s+(-?\d+)\s+?(\S*)\s+(.*)',
+ line.rstrip(), flags=re.VERBOSE)
+ if not m:
+ continue
+ name, type, prio, location, dispname = m.groups()
+ if ':' not in type:
+ # wrong type value. type should be in the form of "{domain}:{objtype}"
+ #
+ # Note: To avoid the regex DoS, this is implemented in python (refs: #8175)
+ continue
+ if type == 'py:module' and type in invdata and name in invdata[type]:
+ # due to a bug in 1.1 and below,
+ # two inventory entries are created
+ # for Python modules, and the first
+ # one is correct
+ continue
+ if type in {'std:label', 'std:term'}:
+ # Some types require case insensitive matches:
+ # * 'term': https://github.com/sphinx-doc/sphinx/issues/9291
+ # * 'label': https://github.com/sphinx-doc/sphinx/issues/12008
+ definition = f"{type}:{name}"
+ content = prio, location, dispname
+ lowercase_definition = definition.lower()
+ if lowercase_definition in potential_ambiguities:
+ if potential_ambiguities[lowercase_definition] != content:
+ actual_ambiguities.add(definition)
+ else:
+ logger.debug(__("inventory <%s> contains duplicate definitions of %s"),
+ uri, definition, type='intersphinx', subtype='external')
+ else:
+ potential_ambiguities[lowercase_definition] = content
+ if location.endswith('$'):
+ location = location[:-1] + name
+ location = join(uri, location)
+ inv_item: InventoryItem = projname, version, location, dispname
+ invdata.setdefault(type, {})[name] = inv_item
+ for ambiguity in actual_ambiguities:
+ logger.info(__("inventory <%s> contains multiple definitions for %s"),
+ uri, ambiguity, type='intersphinx', subtype='external')
+ return invdata
+
+ @classmethod
+ def dump(
+ cls: type[InventoryFile], filename: str, env: BuildEnvironment, builder: Builder,
+ ) -> None:
+ def escape(string: str) -> str:
+ return re.sub("\\s+", " ", string)
+
+ with open(os.path.join(filename), 'wb') as f:
+ # header
+ f.write(('# Sphinx inventory version 2\n'
+ '# Project: %s\n'
+ '# Version: %s\n'
+ '# The remainder of this file is compressed using zlib.\n' %
+ (escape(env.config.project),
+ escape(env.config.version))).encode())
+
+ # body
+ compressor = zlib.compressobj(9)
+ for domainname, domain in sorted(env.domains.items()):
+ for name, dispname, typ, docname, anchor, prio in \
+ sorted(domain.get_objects()):
+ if anchor.endswith(name):
+ # this can shorten the inventory by as much as 25%
+ anchor = anchor[:-len(name)] + '$'
+ uri = builder.get_target_uri(docname)
+ if anchor:
+ uri += '#' + anchor
+ if dispname == name:
+ dispname = '-'
+ entry = ('%s %s:%s %s %s %s\n' %
+ (name, domainname, typ, prio, uri, dispname))
+ f.write(compressor.compress(entry.encode()))
+ f.write(compressor.flush())
diff --git a/sphinx/util/logging.py b/sphinx/util/logging.py
index 00c940a4c..804ef62cc 100644
--- a/sphinx/util/logging.py
+++ b/sphinx/util/logging.py
@@ -1,33 +1,56 @@
"""Logging utility functions for Sphinx."""
+
from __future__ import annotations
+
import logging
import logging.handlers
from collections import defaultdict
from contextlib import contextmanager, nullcontext
from typing import IO, TYPE_CHECKING, Any
+
from docutils import nodes
from docutils.utils import get_source_line
+
from sphinx.errors import SphinxWarning
from sphinx.util.console import colorize
from sphinx.util.osutil import abspath
+
if TYPE_CHECKING:
from collections.abc import Iterator, Sequence, Set
from typing import NoReturn
+
from docutils.nodes import Node
+
from sphinx.application import Sphinx
+
+
NAMESPACE = 'sphinx'
VERBOSE = 15
-LEVEL_NAMES: defaultdict[str, int] = defaultdict(lambda : logging.WARNING,
- {'CRITICAL': logging.CRITICAL, 'SEVERE': logging.CRITICAL, 'ERROR':
- logging.ERROR, 'WARNING': logging.WARNING, 'INFO': logging.INFO,
- 'VERBOSE': VERBOSE, 'DEBUG': logging.DEBUG})
-VERBOSITY_MAP: defaultdict[int, int] = defaultdict(lambda : logging.NOTSET,
- {(0): logging.INFO, (1): VERBOSE, (2): logging.DEBUG})
-COLOR_MAP: defaultdict[int, str] = defaultdict(lambda : 'blue', {logging.
- ERROR: 'darkred', logging.WARNING: 'red', logging.DEBUG: 'darkgray'})
-
-def getLogger(name: str) ->SphinxLoggerAdapter:
+LEVEL_NAMES: defaultdict[str, int] = defaultdict(lambda: logging.WARNING, {
+ 'CRITICAL': logging.CRITICAL,
+ 'SEVERE': logging.CRITICAL,
+ 'ERROR': logging.ERROR,
+ 'WARNING': logging.WARNING,
+ 'INFO': logging.INFO,
+ 'VERBOSE': VERBOSE,
+ 'DEBUG': logging.DEBUG,
+})
+
+VERBOSITY_MAP: defaultdict[int, int] = defaultdict(lambda: logging.NOTSET, {
+ 0: logging.INFO,
+ 1: VERBOSE,
+ 2: logging.DEBUG,
+})
+
+COLOR_MAP: defaultdict[int, str] = defaultdict(lambda: 'blue', {
+ logging.ERROR: 'darkred',
+ logging.WARNING: 'red',
+ logging.DEBUG: 'darkgray',
+})
+
+
+def getLogger(name: str) -> SphinxLoggerAdapter:
"""Get logger wrapped by :class:`sphinx.util.logging.SphinxLoggerAdapter`.
Sphinx logger always uses ``sphinx.*`` namespace to be independent from
@@ -41,37 +64,102 @@ def getLogger(name: str) ->SphinxLoggerAdapter:
>>> logger.info('Hello, this is an extension!')
Hello, this is an extension!
"""
- pass
+ # add sphinx prefix to name forcely
+ logger = logging.getLogger(NAMESPACE + '.' + name)
+ # Forcely enable logger
+ logger.disabled = False
+ # wrap logger by SphinxLoggerAdapter
+ return SphinxLoggerAdapter(logger, {})
-def convert_serializable(records: list[logging.LogRecord]) ->None:
+def convert_serializable(records: list[logging.LogRecord]) -> None:
"""Convert LogRecord serializable."""
- pass
+ for r in records:
+ # extract arguments to a message and clear them
+ r.msg = r.getMessage()
+ r.args = ()
+
+ location = getattr(r, 'location', None)
+ if isinstance(location, nodes.Node):
+ r.location = get_node_location(location)
class SphinxLogRecord(logging.LogRecord):
"""Log record class supporting location"""
+
prefix = ''
location: Any = None
+ def getMessage(self) -> str:
+ message = super().getMessage()
+ location = getattr(self, 'location', None)
+ if location:
+ message = f'{location}: {self.prefix}{message}'
+ elif self.prefix not in message:
+ message = self.prefix + message
+
+ return message
+
class SphinxInfoLogRecord(SphinxLogRecord):
"""Info log record class supporting location"""
- prefix = ''
+
+ prefix = '' # do not show any prefix for INFO messages
class SphinxWarningLogRecord(SphinxLogRecord):
"""Warning log record class supporting location"""
+ @property
+ def prefix(self) -> str: # type: ignore[override]
+ if self.levelno >= logging.CRITICAL:
+ return 'CRITICAL: '
+ elif self.levelno >= logging.ERROR:
+ return 'ERROR: '
+ else:
+ return 'WARNING: '
+
class SphinxLoggerAdapter(logging.LoggerAdapter):
"""LoggerAdapter allowing ``type`` and ``subtype`` keywords."""
+
KEYWORDS = ['type', 'subtype', 'location', 'nonl', 'color', 'once']
- def warning(self, msg: object, *args: object, type: (None | str)=None,
- subtype: (None | str)=None, location: (None | str | tuple[str |
- None, int | None] | Node)=None, nonl: bool=True, color: (str | None
- )=None, once: bool=False, **kwargs: Any) ->None:
+ def log( # type: ignore[override]
+ self, level: int | str, msg: str, *args: Any, **kwargs: Any,
+ ) -> None:
+ if isinstance(level, int):
+ super().log(level, msg, *args, **kwargs)
+ else:
+ levelno = LEVEL_NAMES[level]
+ super().log(levelno, msg, *args, **kwargs)
+
+ def verbose(self, msg: str, *args: Any, **kwargs: Any) -> None:
+ self.log(VERBOSE, msg, *args, **kwargs)
+
+ def process(self, msg: str, kwargs: dict) -> tuple[str, dict]: # type: ignore[override]
+ extra = kwargs.setdefault('extra', {})
+ for keyword in self.KEYWORDS:
+ if keyword in kwargs:
+ extra[keyword] = kwargs.pop(keyword)
+
+ return msg, kwargs
+
+ def handle(self, record: logging.LogRecord) -> None:
+ self.logger.handle(record)
+
+ def warning( # type: ignore[override]
+ self,
+ msg: object,
+ *args: object,
+ type: None | str = None,
+ subtype: None | str = None,
+ location: None | str | tuple[str | None, int | None] | Node = None,
+ nonl: bool = True,
+ color: str | None = None,
+ once: bool = False,
+ **kwargs: Any,
+ ) -> None:
"""Log a sphinx warning.
It is recommended to include a ``type`` and ``subtype`` for warnings as
@@ -94,37 +182,100 @@ class SphinxLoggerAdapter(logging.LoggerAdapter):
:param once: Do not log this warning,
if a previous warning already has same ``msg``, ``args`` and ``once=True``.
"""
- pass
+ return super().warning(
+ msg,
+ *args,
+ type=type,
+ subtype=subtype,
+ location=location,
+ nonl=nonl,
+ color=color,
+ once=once,
+ **kwargs,
+ )
class WarningStreamHandler(logging.StreamHandler):
"""StreamHandler for warnings."""
+
pass
class NewLineStreamHandler(logging.StreamHandler):
"""StreamHandler which switches line terminator by record.nonl flag."""
+ def emit(self, record: logging.LogRecord) -> None:
+ try:
+ self.acquire()
+ if getattr(record, 'nonl', False):
+ # skip appending terminator when nonl=True
+ self.terminator = ''
+ super().emit(record)
+ finally:
+ self.terminator = '\n'
+ self.release()
+
class MemoryHandler(logging.handlers.BufferingHandler):
"""Handler buffering all logs."""
+
buffer: list[logging.LogRecord]
- def __init__(self) ->None:
+ def __init__(self) -> None:
super().__init__(-1)
+ def shouldFlush(self, record: logging.LogRecord) -> bool:
+ return False # never flush
+
+ def flush(self) -> None:
+ # suppress any flushes triggered by importing packages that flush
+ # all handlers at initialization time
+ pass
+
+ def flushTo(self, logger: logging.Logger) -> None:
+ self.acquire()
+ try:
+ for record in self.buffer:
+ logger.handle(record)
+ self.buffer = []
+ finally:
+ self.release()
+
+ def clear(self) -> list[logging.LogRecord]:
+ buffer, self.buffer = self.buffer, []
+ return buffer
+
@contextmanager
-def pending_warnings() ->Iterator[logging.Handler]:
+def pending_warnings() -> Iterator[logging.Handler]:
"""Context manager to postpone logging warnings temporarily.
Similar to :func:`pending_logging`.
"""
- pass
+ logger = logging.getLogger(NAMESPACE)
+ memhandler = MemoryHandler()
+ memhandler.setLevel(logging.WARNING)
+
+ try:
+ handlers = []
+ for handler in logger.handlers[:]:
+ if isinstance(handler, WarningStreamHandler):
+ logger.removeHandler(handler)
+ handlers.append(handler)
+
+ logger.addHandler(memhandler)
+ yield memhandler
+ finally:
+ logger.removeHandler(memhandler)
+
+ for handler in handlers:
+ logger.addHandler(handler)
+
+ memhandler.flushTo(logger)
@contextmanager
-def suppress_logging() ->Iterator[MemoryHandler]:
+def suppress_logging() -> Iterator[MemoryHandler]:
"""Context manager to suppress logging all logs temporarily.
For example::
@@ -134,11 +285,26 @@ def suppress_logging() ->Iterator[MemoryHandler]:
>>> some_long_process()
>>>
"""
- pass
+ logger = logging.getLogger(NAMESPACE)
+ memhandler = MemoryHandler()
+
+ try:
+ handlers = []
+ for handler in logger.handlers[:]:
+ logger.removeHandler(handler)
+ handlers.append(handler)
+
+ logger.addHandler(memhandler)
+ yield memhandler
+ finally:
+ logger.removeHandler(memhandler)
+
+ for handler in handlers:
+ logger.addHandler(handler)
@contextmanager
-def pending_logging() ->Iterator[MemoryHandler]:
+def pending_logging() -> Iterator[MemoryHandler]:
"""Context manager to postpone logging all logs temporarily.
For example::
@@ -149,14 +315,19 @@ def pending_logging() ->Iterator[MemoryHandler]:
>>>
Warning message! # the warning is flushed here
"""
- pass
+ logger = logging.getLogger(NAMESPACE)
+ try:
+ with suppress_logging() as memhandler:
+ yield memhandler
+ finally:
+ memhandler.flushTo(logger)
-skip_warningiserror = nullcontext
+skip_warningiserror = nullcontext # Deprecate in Sphinx 10
@contextmanager
-def prefixed_warnings(prefix: str) ->Iterator[None]:
+def prefixed_warnings(prefix: str) -> Iterator[None]:
"""Context manager to prepend prefix to all warning log records temporarily.
For example::
@@ -166,52 +337,145 @@ def prefixed_warnings(prefix: str) ->Iterator[None]:
.. versionadded:: 2.0
"""
- pass
+ logger = logging.getLogger(NAMESPACE)
+ warning_handler = None
+ for handler in logger.handlers:
+ if isinstance(handler, WarningStreamHandler):
+ warning_handler = handler
+ break
+ else:
+ # warning stream not found
+ yield
+ return
+
+ prefix_filter = None
+ for _filter in warning_handler.filters:
+ if isinstance(_filter, MessagePrefixFilter):
+ prefix_filter = _filter
+ break
+
+ if prefix_filter:
+ # already prefixed
+ try:
+ previous = prefix_filter.prefix
+ prefix_filter.prefix = prefix
+ yield
+ finally:
+ prefix_filter.prefix = previous
+ else:
+ # not prefixed yet
+ prefix_filter = MessagePrefixFilter(prefix)
+ try:
+ warning_handler.addFilter(prefix_filter)
+ yield
+ finally:
+ warning_handler.removeFilter(prefix_filter)
class LogCollector:
-
- def __init__(self) ->None:
+ def __init__(self) -> None:
self.logs: list[logging.LogRecord] = []
+ @contextmanager
+ def collect(self) -> Iterator[None]:
+ with pending_logging() as memhandler:
+ yield
+
+ self.logs = memhandler.clear()
+
class InfoFilter(logging.Filter):
"""Filter error and warning messages."""
+ def filter(self, record: logging.LogRecord) -> bool:
+ return record.levelno < logging.WARNING
+
class _RaiseOnWarningFilter(logging.Filter):
"""Raise exception if a warning is emitted."""
-
-def is_suppressed_warning(warning_type: str, sub_type: str,
- suppress_warnings: (Set[str] | Sequence[str])) ->bool:
+ def filter(self, record: logging.LogRecord) -> NoReturn:
+ try:
+ message = record.msg % record.args
+ except (TypeError, ValueError):
+ message = record.msg # use record.msg itself
+ if location := getattr(record, 'location', ''):
+ message = f"{location}:{message}"
+ if record.exc_info is not None:
+ raise SphinxWarning(message) from record.exc_info[1]
+ raise SphinxWarning(message)
+
+
+def is_suppressed_warning(
+ warning_type: str, sub_type: str, suppress_warnings: Set[str] | Sequence[str],
+) -> bool:
"""Check whether the warning is suppressed or not."""
- pass
+ if warning_type is None or len(suppress_warnings) == 0:
+ return False
+ suppressed_warnings = frozenset(suppress_warnings)
+ if warning_type in suppressed_warnings:
+ return True
+ if f'{warning_type}.*' in suppressed_warnings:
+ return True
+ return f'{warning_type}.{sub_type}' in suppressed_warnings
class WarningSuppressor(logging.Filter):
"""Filter logs by `suppress_warnings`."""
- def __init__(self, app: Sphinx) ->None:
+ def __init__(self, app: Sphinx) -> None:
self.app = app
super().__init__()
+ def filter(self, record: logging.LogRecord) -> bool:
+ type = getattr(record, 'type', '')
+ subtype = getattr(record, 'subtype', '')
+
+ try:
+ suppress_warnings = self.app.config.suppress_warnings
+ except AttributeError:
+ # config is not initialized yet (ex. in conf.py)
+ suppress_warnings = ()
+
+ if is_suppressed_warning(type, subtype, suppress_warnings):
+ return False
+ else:
+ self.app._warncount += 1
+ return True
+
class MessagePrefixFilter(logging.Filter):
"""Prepend prefix to all log records."""
- def __init__(self, prefix: str) ->None:
+ def __init__(self, prefix: str) -> None:
self.prefix = prefix
super().__init__()
+ def filter(self, record: logging.LogRecord) -> bool:
+ if self.prefix:
+ record.msg = self.prefix + ' ' + record.msg
+ return True
+
class OnceFilter(logging.Filter):
"""Show the message only once."""
- def __init__(self, name: str='') ->None:
+ def __init__(self, name: str = '') -> None:
super().__init__(name)
self.messages: dict[str, list] = {}
+ def filter(self, record: logging.LogRecord) -> bool:
+ once = getattr(record, 'once', '')
+ if not once:
+ return True
+ else:
+ params = self.messages.setdefault(record.msg, [])
+ if record.args in params:
+ return False
+
+ params.append(record.args)
+ return True
+
class SphinxLogRecordTranslator(logging.Filter):
"""Converts a log record to one Sphinx expects
@@ -220,42 +484,148 @@ class SphinxLogRecordTranslator(logging.Filter):
* docname to path if location given
* append warning type/subtype to message if :confval:`show_warning_types` is ``True``
"""
+
LogRecordClass: type[logging.LogRecord]
- def __init__(self, app: Sphinx) ->None:
+ def __init__(self, app: Sphinx) -> None:
self.app = app
super().__init__()
+ def filter(self, record: SphinxWarningLogRecord) -> bool: # type: ignore[override]
+ if isinstance(record, logging.LogRecord):
+ # force subclassing to handle location
+ record.__class__ = self.LogRecordClass # type: ignore[assignment]
+
+ location = getattr(record, 'location', None)
+ if isinstance(location, tuple):
+ docname, lineno = location
+ if docname:
+ if lineno:
+ record.location = f'{self.app.env.doc2path(docname)}:{lineno}'
+ else:
+ record.location = f'{self.app.env.doc2path(docname)}'
+ else:
+ record.location = None
+ elif isinstance(location, nodes.Node):
+ record.location = get_node_location(location)
+ elif location and ':' not in location:
+ record.location = f'{self.app.env.doc2path(location)}'
+
+ return True
+
class InfoLogRecordTranslator(SphinxLogRecordTranslator):
"""LogRecordTranslator for INFO level log records."""
+
LogRecordClass = SphinxInfoLogRecord
class WarningLogRecordTranslator(SphinxLogRecordTranslator):
"""LogRecordTranslator for WARNING level log records."""
+
LogRecordClass = SphinxWarningLogRecord
+ def filter(self, record: SphinxWarningLogRecord) -> bool: # type: ignore[override]
+ ret = super().filter(record)
+
+ try:
+ show_warning_types = self.app.config.show_warning_types
+ except AttributeError:
+ # config is not initialized yet (ex. in conf.py)
+ show_warning_types = False
+ if show_warning_types:
+ if log_type := getattr(record, 'type', ''):
+ if log_subtype := getattr(record, 'subtype', ''):
+ record.msg += f' [{log_type}.{log_subtype}]'
+ else:
+ record.msg += f' [{log_type}]'
+
+ return ret
+
+
+def get_node_location(node: Node) -> str | None:
+ source, line = get_source_line(node)
+ if source and line:
+ return f"{abspath(source)}:{line}"
+ if source:
+ return f"{abspath(source)}:"
+ if line:
+ return f"<unknown>:{line}"
+ return None
+
class ColorizeFormatter(logging.Formatter):
- pass
+ def format(self, record: logging.LogRecord) -> str:
+ message = super().format(record)
+ color = getattr(record, 'color', None)
+ if color is None:
+ color = COLOR_MAP.get(record.levelno)
+
+ if color:
+ return colorize(color, message)
+ else:
+ return message
class SafeEncodingWriter:
"""Stream writer which ignores UnicodeEncodeError silently"""
- def __init__(self, stream: IO) ->None:
+ def __init__(self, stream: IO) -> None:
self.stream = stream
self.encoding = getattr(stream, 'encoding', 'ascii') or 'ascii'
+ def write(self, data: str) -> None:
+ try:
+ self.stream.write(data)
+ except UnicodeEncodeError:
+ # stream accept only str, not bytes. So, we encode and replace
+ # non-encodable characters, then decode them.
+ self.stream.write(data.encode(self.encoding, 'replace').decode(self.encoding))
+
+ def flush(self) -> None:
+ if hasattr(self.stream, 'flush'):
+ self.stream.flush()
+
class LastMessagesWriter:
"""Stream writer storing last 10 messages in memory to save trackback"""
- def __init__(self, app: Sphinx, stream: IO) ->None:
+ def __init__(self, app: Sphinx, stream: IO) -> None:
self.app = app
+ def write(self, data: str) -> None:
+ self.app.messagelog.append(data)
-def setup(app: Sphinx, status: IO, warning: IO) ->None:
+
+def setup(app: Sphinx, status: IO, warning: IO) -> None:
"""Setup root logger for Sphinx"""
- pass
+ logger = logging.getLogger(NAMESPACE)
+ logger.setLevel(logging.DEBUG)
+ logger.propagate = False
+
+ # clear all handlers
+ for handler in logger.handlers[:]:
+ logger.removeHandler(handler)
+
+ info_handler = NewLineStreamHandler(SafeEncodingWriter(status))
+ info_handler.addFilter(InfoFilter())
+ info_handler.addFilter(InfoLogRecordTranslator(app))
+ info_handler.setLevel(VERBOSITY_MAP[app.verbosity])
+ info_handler.setFormatter(ColorizeFormatter())
+
+ warning_handler = WarningStreamHandler(SafeEncodingWriter(warning))
+ if app._exception_on_warning:
+ warning_handler.addFilter(_RaiseOnWarningFilter())
+ warning_handler.addFilter(WarningSuppressor(app))
+ warning_handler.addFilter(WarningLogRecordTranslator(app))
+ warning_handler.addFilter(OnceFilter())
+ warning_handler.setLevel(logging.WARNING)
+ warning_handler.setFormatter(ColorizeFormatter())
+
+ messagelog_handler = logging.StreamHandler(LastMessagesWriter(app, status))
+ messagelog_handler.addFilter(InfoFilter())
+ messagelog_handler.setLevel(VERBOSITY_MAP[app.verbosity])
+
+ logger.addHandler(info_handler)
+ logger.addHandler(warning_handler)
+ logger.addHandler(messagelog_handler)
diff --git a/sphinx/util/matching.py b/sphinx/util/matching.py
index de4967a6a..79c56dd2e 100644
--- a/sphinx/util/matching.py
+++ b/sphinx/util/matching.py
@@ -1,20 +1,67 @@
"""Pattern-matching utility functions for Sphinx."""
+
from __future__ import annotations
+
import os.path
import re
from typing import TYPE_CHECKING
+
from sphinx.util.osutil import canon_path, path_stabilize
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Iterator
-def _translate_pattern(pat: str) ->str:
+def _translate_pattern(pat: str) -> str:
"""Translate a shell-style glob pattern to a regular expression.
Adapted from the fnmatch module, but enhanced so that single stars don't
match slashes.
"""
- pass
+ i, n = 0, len(pat)
+ res = ''
+ while i < n:
+ c = pat[i]
+ i += 1
+ if c == '*':
+ if i < n and pat[i] == '*':
+ # double star matches slashes too
+ i += 1
+ res = res + '.*'
+ else:
+ # single star doesn't match slashes
+ res = res + '[^/]*'
+ elif c == '?':
+ # question mark doesn't match slashes too
+ res = res + '[^/]'
+ elif c == '[':
+ j = i
+ if j < n and pat[j] == '!':
+ j += 1
+ if j < n and pat[j] == ']':
+ j += 1
+ while j < n and pat[j] != ']':
+ j += 1
+ if j >= n:
+ res = res + '\\['
+ else:
+ stuff = pat[i:j].replace('\\', '\\\\')
+ i = j + 1
+ if stuff[0] == '!':
+ # negative pattern mustn't match slashes too
+ stuff = '^/' + stuff[1:]
+ elif stuff[0] == '^':
+ stuff = '\\' + stuff
+ res = f'{res}[{stuff}]'
+ else:
+ res += re.escape(c)
+ return res + '$'
+
+
+def compile_matchers(
+ patterns: Iterable[str],
+) -> list[Callable[[str], re.Match[str] | None]]:
+ return [re.compile(_translate_pattern(pat)).match for pat in patterns]
class Matcher:
@@ -24,37 +71,50 @@ class Matcher:
For example, "**/index.rst" matches with "index.rst"
"""
- def __init__(self, exclude_patterns: Iterable[str]) ->None:
- expanded = [pat[3:] for pat in exclude_patterns if pat.startswith(
- '**/')]
+ def __init__(self, exclude_patterns: Iterable[str]) -> None:
+ expanded = [pat[3:] for pat in exclude_patterns if pat.startswith('**/')]
self.patterns = compile_matchers(list(exclude_patterns) + expanded)
- def __call__(self, string: str) ->bool:
+ def __call__(self, string: str) -> bool:
return self.match(string)
+ def match(self, string: str) -> bool:
+ string = canon_path(string)
+ return any(pat(string) for pat in self.patterns)
+
DOTFILES = Matcher(['**/.*'])
+
+
_pat_cache: dict[str, re.Pattern[str]] = {}
-def patmatch(name: str, pat: str) ->(re.Match[str] | None):
+def patmatch(name: str, pat: str) -> re.Match[str] | None:
"""Return if name matches the regular expression (pattern)
``pat```. Adapted from fnmatch module.
"""
- pass
+ if pat not in _pat_cache:
+ _pat_cache[pat] = re.compile(_translate_pattern(pat))
+ return _pat_cache[pat].match(name)
-def patfilter(names: Iterable[str], pat: str) ->list[str]:
+def patfilter(names: Iterable[str], pat: str) -> list[str]:
"""Return the subset of the list ``names`` that match
the regular expression (pattern) ``pat``.
Adapted from fnmatch module.
"""
- pass
+ if pat not in _pat_cache:
+ _pat_cache[pat] = re.compile(_translate_pattern(pat))
+ match = _pat_cache[pat].match
+ return list(filter(match, names))
-def get_matching_files(dirname: (str | os.PathLike[str]), include_patterns:
- Iterable[str]=('**',), exclude_patterns: Iterable[str]=()) ->Iterator[str]:
+def get_matching_files(
+ dirname: str | os.PathLike[str],
+ include_patterns: Iterable[str] = ("**",),
+ exclude_patterns: Iterable[str] = (),
+) -> Iterator[str]:
"""Get all file names in a directory, recursively.
Filter file names by the glob-style include_patterns and exclude_patterns.
@@ -64,4 +124,47 @@ def get_matching_files(dirname: (str | os.PathLike[str]), include_patterns:
exclusions from *exclude_patterns* take priority over inclusions.
"""
- pass
+ # dirname is a normalized absolute path.
+ dirname = os.path.normpath(os.path.abspath(dirname))
+
+ exclude_matchers = compile_matchers(exclude_patterns)
+ include_matchers = compile_matchers(include_patterns)
+
+ for root, dirs, files in os.walk(dirname, followlinks=True):
+ relative_root = os.path.relpath(root, dirname)
+ if relative_root == ".":
+ relative_root = "" # suppress dirname for files on the target dir
+
+ # Filter files
+ included_files = []
+ for entry in sorted(files):
+ entry = path_stabilize(os.path.join(relative_root, entry))
+ keep = False
+ for matcher in include_matchers:
+ if matcher(entry):
+ keep = True
+ break # break the inner loop
+
+ for matcher in exclude_matchers:
+ if matcher(entry):
+ keep = False
+ break # break the inner loop
+
+ if keep:
+ included_files.append(entry)
+
+ # Filter directories
+ filtered_dirs = []
+ for dir_name in sorted(dirs):
+ normalised = path_stabilize(os.path.join(relative_root, dir_name))
+ for matcher in exclude_matchers:
+ if matcher(normalised):
+ break # break the inner loop
+ else:
+ # if the loop didn't break
+ filtered_dirs.append(dir_name)
+
+ dirs[:] = filtered_dirs
+
+ # Yield filtered files
+ yield from included_files
diff --git a/sphinx/util/math.py b/sphinx/util/math.py
index 63ddbabd2..576fdcf53 100644
--- a/sphinx/util/math.py
+++ b/sphinx/util/math.py
@@ -1,6 +1,62 @@
"""Utility functions for math."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING
+
if TYPE_CHECKING:
from docutils import nodes
+
from sphinx.writers.html5 import HTML5Translator
+
+
+def get_node_equation_number(writer: HTML5Translator, node: nodes.math_block) -> str:
+ if writer.builder.config.math_numfig and writer.builder.config.numfig:
+ figtype = 'displaymath'
+ if writer.builder.name == 'singlehtml':
+ key = f"{writer.docnames[-1]}/{figtype}" # type: ignore[has-type]
+ else:
+ key = figtype
+
+ id = node['ids'][0]
+ number = writer.builder.fignumbers.get(key, {}).get(id, ())
+ eqno = '.'.join(map(str, number))
+ eqno = writer.builder.config.math_numsep.join(eqno.rsplit('.', 1))
+ return eqno
+ else:
+ return node['number']
+
+
+def wrap_displaymath(text: str, label: str | None, numbering: bool) -> str:
+ def is_equation(part: str) -> str:
+ return part.strip()
+
+ if label is None:
+ labeldef = ''
+ else:
+ labeldef = r'\label{%s}' % label
+ numbering = True
+
+ parts = list(filter(is_equation, text.split('\n\n')))
+ equations = []
+ if len(parts) == 0:
+ return ''
+ elif len(parts) == 1:
+ if numbering:
+ begin = r'\begin{equation}' + labeldef
+ end = r'\end{equation}'
+ else:
+ begin = r'\begin{equation*}' + labeldef
+ end = r'\end{equation*}'
+ equations.append('\\begin{split}%s\\end{split}\n' % parts[0])
+ else:
+ if numbering:
+ begin = r'\begin{align}%s\!\begin{aligned}' % labeldef
+ end = r'\end{aligned}\end{align}'
+ else:
+ begin = r'\begin{align*}%s\!\begin{aligned}' % labeldef
+ end = r'\end{aligned}\end{align*}'
+ equations.extend('%s\\\\\n' % part.strip() for part in parts)
+
+ concatenated_equations = ''.join(equations)
+ return f'{begin}\n{concatenated_equations}{end}'
diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py
index c491cf3fb..01ce81290 100644
--- a/sphinx/util/nodes.py
+++ b/sphinx/util/nodes.py
@@ -1,28 +1,41 @@
"""Docutils node-related utility functions for Sphinx."""
+
from __future__ import annotations
+
import contextlib
import re
import unicodedata
from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast
+
from docutils import nodes
from docutils.nodes import Node
+
from sphinx import addnodes
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.parsing import _fresh_title_style_context
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Iterator
+
from docutils.nodes import Element
from docutils.parsers.rst import Directive
from docutils.parsers.rst.states import Inliner, RSTState
from docutils.statemachine import StringList
+
from sphinx.builders import Builder
from sphinx.environment import BuildEnvironment
from sphinx.util.tags import Tags
+
logger = logging.getLogger(__name__)
-explicit_title_re = re.compile('^(.+?)\\s*(?<!\\x00)<([^<]*?)>$', re.DOTALL)
-caption_ref_re = explicit_title_re
-N = TypeVar('N', bound=Node)
+
+
+# \x00 means the "<" was backslash-escaped
+explicit_title_re = re.compile(r'^(.+?)\s*(?<!\x00)<([^<]*?)>$', re.DOTALL)
+caption_ref_re = explicit_title_re # b/w compat alias
+
+
+N = TypeVar("N", bound=Node)
class NodeMatcher(Generic[N]):
@@ -46,33 +59,56 @@ class NodeMatcher(Generic[N]):
# => [<reference ...>, <reference ...>, ...]
"""
- def __init__(self, *node_classes: type[N], **attrs: Any) ->None:
+ def __init__(self, *node_classes: type[N], **attrs: Any) -> None:
self.classes = node_classes
self.attrs = attrs
- def __call__(self, node: Node) ->bool:
+ def match(self, node: Node) -> bool:
+ try:
+ if self.classes and not isinstance(node, self.classes):
+ return False
+
+ if self.attrs:
+ if not isinstance(node, nodes.Element):
+ return False
+
+ for key, value in self.attrs.items():
+ if key not in node:
+ return False
+ elif value is Any:
+ continue
+ elif node.get(key) != value:
+ return False
+
+ return True
+ except Exception:
+ # for non-Element nodes
+ return False
+
+ def __call__(self, node: Node) -> bool:
return self.match(node)
- def findall(self, node: Node) ->Iterator[N]:
+ def findall(self, node: Node) -> Iterator[N]:
"""An alternative to `Node.findall` with improved type safety.
While the `NodeMatcher` object can be used as an argument to `Node.findall`, doing so
confounds type checkers' ability to determine the return type of the iterator.
"""
- pass
+ for found in node.findall(self):
+ yield cast(N, found)
-def get_full_module_name(node: Node) ->str:
+def get_full_module_name(node: Node) -> str:
"""
Return full module dotted path like: 'docutils.nodes.paragraph'
:param nodes.Node node: target node
:return: full module dotted path
"""
- pass
+ return f'{node.__module__}.{node.__class__.__name__}'
-def repr_domxml(node: Node, length: int=80) ->str:
+def repr_domxml(node: Node, length: int = 80) -> str:
"""
return DOM XML representation of the specified node like:
'<paragraph translatable="False"><inline classes="versionadded">Added in version...'
@@ -83,29 +119,214 @@ def repr_domxml(node: Node, length: int=80) ->str:
returns full of DOM XML representation.
:return: DOM XML representation
"""
- pass
-
-
-IGNORED_NODES = (nodes.Invisible, nodes.literal_block, nodes.doctest_block,
- addnodes.versionmodified)
-LITERAL_TYPE_NODES = (nodes.literal_block, nodes.doctest_block, nodes.
- math_block, nodes.raw)
-IMAGE_TYPE_NODES = nodes.image,
-
-
-def extract_messages(doctree: Element) ->Iterable[tuple[Element, str]]:
+ try:
+ text = node.asdom().toxml()
+ except Exception:
+ text = str(node)
+ if length and len(text) > length:
+ text = text[:length] + '...'
+ return text
+
+
+def apply_source_workaround(node: Element) -> None:
+ # workaround: nodes.term have wrong rawsource if classifier is specified.
+ # The behavior of docutils-0.11, 0.12 is:
+ # * when ``term text : classifier1 : classifier2`` is specified,
+ # * rawsource of term node will have: ``term text : classifier1 : classifier2``
+ # * rawsource of classifier node will be None
+ if isinstance(node, nodes.classifier) and not node.rawsource:
+ logger.debug('[i18n] PATCH: %r to have source, line and rawsource: %s',
+ get_full_module_name(node), repr_domxml(node))
+ definition_list_item = node.parent
+ node.source = definition_list_item.source
+ node.line = definition_list_item.line - 1 # type: ignore[operator]
+ node.rawsource = node.astext() # set 'classifier1' (or 'classifier2')
+ elif isinstance(node, nodes.classifier) and not node.source:
+ # docutils-0.15 fills in rawsource attribute, but not in source.
+ node.source = node.parent.source
+ if isinstance(node, nodes.image) and node.source is None:
+ logger.debug('[i18n] PATCH: %r to have source, line: %s',
+ get_full_module_name(node), repr_domxml(node))
+ node.source, node.line = node.parent.source, node.parent.line
+ if isinstance(node, nodes.title) and node.source is None:
+ logger.debug('[i18n] PATCH: %r to have source: %s',
+ get_full_module_name(node), repr_domxml(node))
+ node.source, node.line = node.parent.source, node.parent.line
+ if isinstance(node, nodes.term):
+ logger.debug('[i18n] PATCH: %r to have rawsource: %s',
+ get_full_module_name(node), repr_domxml(node))
+ # strip classifier from rawsource of term
+ for classifier in reversed(list(node.parent.findall(nodes.classifier))):
+ node.rawsource = re.sub(r'\s*:\s*%s' % re.escape(classifier.astext()),
+ '', node.rawsource)
+ if isinstance(node, nodes.topic) and node.source is None:
+ # docutils-0.18 does not fill the source attribute of topic
+ logger.debug('[i18n] PATCH: %r to have source, line: %s',
+ get_full_module_name(node), repr_domxml(node))
+ node.source, node.line = node.parent.source, node.parent.line
+
+ # workaround: literal_block under bullet list (#4913)
+ if isinstance(node, nodes.literal_block) and node.source is None:
+ with contextlib.suppress(ValueError):
+ node.source = get_node_source(node)
+
+ # workaround: recommonmark-0.2.0 doesn't set rawsource attribute
+ if not node.rawsource:
+ node.rawsource = node.astext()
+
+ if node.source and node.rawsource:
+ return
+
+ # workaround: some docutils nodes doesn't have source, line.
+ if isinstance(node, (
+ nodes.rubric # #1305 rubric directive
+ | nodes.line # #1477 line node
+ | nodes.image # #3093 image directive in substitution
+ | nodes.field_name # #3335 field list syntax
+ )):
+ logger.debug('[i18n] PATCH: %r to have source and line: %s',
+ get_full_module_name(node), repr_domxml(node))
+ try:
+ node.source = get_node_source(node)
+ except ValueError:
+ node.source = ''
+ node.line = 0 # need fix docutils to get `node.line`
+ return
+
+
+IGNORED_NODES = (
+ nodes.Invisible,
+ nodes.literal_block,
+ nodes.doctest_block,
+ addnodes.versionmodified,
+ # XXX there are probably more
+)
+
+
+def is_translatable(node: Node) -> bool:
+ if isinstance(node, addnodes.translatable):
+ return True
+
+ # image node marked as translatable or having alt text
+ if isinstance(node, nodes.image) and (node.get('translatable') or node.get('alt')):
+ return True
+
+ if isinstance(node, nodes.Inline) and 'translatable' not in node: # type: ignore[operator]
+ # inline node must not be translated if 'translatable' is not set
+ return False
+
+ if isinstance(node, nodes.TextElement):
+ if not node.source:
+ logger.debug('[i18n] SKIP %r because no node.source: %s',
+ get_full_module_name(node), repr_domxml(node))
+ return False # built-in message
+ if isinstance(node, IGNORED_NODES) and 'translatable' not in node:
+ logger.debug("[i18n] SKIP %r because node is in IGNORED_NODES "
+ "and no node['translatable']: %s",
+ get_full_module_name(node), repr_domxml(node))
+ return False
+ if not node.get('translatable', True):
+ # not(node['translatable'] == True or node['translatable'] is None)
+ logger.debug("[i18n] SKIP %r because not node['translatable']: %s",
+ get_full_module_name(node), repr_domxml(node))
+ return False
+ # <field_name>orphan</field_name>
+ # XXX ignore all metadata (== docinfo)
+ if isinstance(node, nodes.field_name) and (node.children[0] == 'orphan'):
+ logger.debug('[i18n] SKIP %r because orphan node: %s',
+ get_full_module_name(node), repr_domxml(node))
+ return False
+ return True
+
+ return isinstance(node, nodes.meta)
+
+
+LITERAL_TYPE_NODES = (
+ nodes.literal_block,
+ nodes.doctest_block,
+ nodes.math_block,
+ nodes.raw,
+)
+IMAGE_TYPE_NODES = (
+ nodes.image,
+)
+
+
+def extract_messages(doctree: Element) -> Iterable[tuple[Element, str]]:
"""Extract translatable messages from a document tree."""
- pass
-
-
-def traverse_translatable_index(doctree: Element) ->Iterable[tuple[Element,
- list[tuple[str, str, str, str, str | None]]]]:
+ for node in doctree.findall(is_translatable):
+ if isinstance(node, addnodes.translatable):
+ for msg in node.extract_original_messages():
+ yield node, msg # type: ignore[misc]
+ continue
+ if isinstance(node, LITERAL_TYPE_NODES):
+ msg = node.rawsource
+ if not msg:
+ msg = node.astext()
+ elif isinstance(node, nodes.image):
+ if node.get('alt'):
+ yield node, node['alt']
+ if node.get('translatable'):
+ image_uri = node.get('original_uri', node['uri'])
+ msg = f'.. image:: {image_uri}'
+ else:
+ msg = ''
+ elif isinstance(node, nodes.meta):
+ msg = node["content"]
+ else:
+ msg = node.rawsource.replace('\n', ' ').strip() # type: ignore[attr-defined]
+
+ # XXX nodes rendering empty are likely a bug in sphinx.addnodes
+ if msg:
+ yield node, msg # type: ignore[misc]
+
+
+def get_node_source(node: Element) -> str:
+ for pnode in traverse_parent(node):
+ if pnode.source:
+ return pnode.source
+ msg = 'node source not found'
+ raise ValueError(msg)
+
+
+def get_node_line(node: Element) -> int:
+ for pnode in traverse_parent(node):
+ if pnode.line:
+ return pnode.line
+ msg = 'node line not found'
+ raise ValueError(msg)
+
+
+def traverse_parent(node: Element, cls: Any = None) -> Iterable[Element]:
+ while node:
+ if cls is None or isinstance(node, cls):
+ yield node
+ node = node.parent
+
+
+def get_prev_node(node: Node) -> Node | None:
+ pos = node.parent.index(node)
+ if pos > 0:
+ return node.parent[pos - 1]
+ else:
+ return None
+
+
+def traverse_translatable_index(
+ doctree: Element,
+) -> Iterable[tuple[Element, list[tuple[str, str, str, str, str | None]]]]:
"""Traverse translatable index node from a document tree."""
- pass
+ matcher = NodeMatcher(addnodes.index, inline=False)
+ for node in matcher.findall(doctree):
+ if 'raw_entries' in node:
+ entries = node['raw_entries']
+ else:
+ entries = node['entries']
+ yield node, entries
-def nested_parse_with_titles(state: RSTState, content: StringList, node:
- Node, content_offset: int=0) ->str:
+def nested_parse_with_titles(state: RSTState, content: StringList, node: Node,
+ content_offset: int = 0) -> str:
"""Version of state.nested_parse() that allows titles and does not require
titles to have the same decoration as the calling document.
@@ -115,33 +336,119 @@ def nested_parse_with_titles(state: RSTState, content: StringList, node:
This function is retained for compatibility and will be deprecated in
Sphinx 8. Prefer ``nested_parse_to_nodes()``.
"""
- pass
+ with _fresh_title_style_context(state):
+ ret = state.nested_parse(content, content_offset, node, match_titles=True)
+ return ret
-def clean_astext(node: Element) ->str:
+def clean_astext(node: Element) -> str:
"""Like node.astext(), but ignore images."""
- pass
+ node = node.deepcopy()
+ for img in node.findall(nodes.image):
+ img['alt'] = ''
+ for raw in list(node.findall(nodes.raw)):
+ raw.parent.remove(raw)
+ return node.astext()
-def split_explicit_title(text: str) ->tuple[bool, str, str]:
+def split_explicit_title(text: str) -> tuple[bool, str, str]:
"""Split role content into title and target, if given."""
- pass
-
-
-indextypes = ['single', 'pair', 'double', 'triple', 'see', 'seealso']
-
-
-def inline_all_toctrees(builder: Builder, docnameset: set[str], docname:
- str, tree: nodes.document, colorfunc: Callable[[str], str], traversed:
- list[str], indent: str='') ->nodes.document:
+ match = explicit_title_re.match(text)
+ if match:
+ return True, match.group(1), match.group(2)
+ return False, text, text
+
+
+indextypes = [
+ 'single', 'pair', 'double', 'triple', 'see', 'seealso',
+]
+
+
+def process_index_entry(entry: str, targetid: str,
+ ) -> list[tuple[str, str, str, str, str | None]]:
+ from sphinx.domains.python import pairindextypes
+
+ indexentries: list[tuple[str, str, str, str, str | None]] = []
+ entry = entry.strip()
+ oentry = entry
+ main = ''
+ if entry.startswith('!'):
+ main = 'main'
+ entry = entry[1:].lstrip()
+ for index_type in pairindextypes:
+ if entry.startswith(f'{index_type}:'):
+ value = entry[len(index_type) + 1:].strip()
+ value = f'{pairindextypes[index_type]}; {value}'
+ # xref RemovedInSphinx90Warning
+ logger.warning(__('%r is deprecated for index entries (from entry %r). '
+ "Use 'pair: %s' instead."),
+ index_type, entry, value, type='index')
+ indexentries.append(('pair', value, targetid, main, None))
+ break
+ else:
+ for index_type in indextypes:
+ if entry.startswith(f'{index_type}:'):
+ value = entry[len(index_type) + 1:].strip()
+ if index_type == 'double':
+ index_type = 'pair'
+ indexentries.append((index_type, value, targetid, main, None))
+ break
+ # shorthand notation for single entries
+ else:
+ for value in oentry.split(','):
+ value = value.strip()
+ main = ''
+ if value.startswith('!'):
+ main = 'main'
+ value = value[1:].lstrip()
+ if not value:
+ continue
+ indexentries.append(('single', value, targetid, main, None))
+ return indexentries
+
+
+def inline_all_toctrees(
+ builder: Builder,
+ docnameset: set[str],
+ docname: str,
+ tree: nodes.document,
+ colorfunc: Callable[[str], str],
+ traversed: list[str],
+ indent: str = '',
+) -> nodes.document:
"""Inline all toctrees in the *tree*.
Record all docnames in *docnameset*, and output docnames with *colorfunc*.
"""
- pass
-
-
-def _make_id(string: str) ->str:
+ tree = tree.deepcopy()
+ for toctreenode in list(tree.findall(addnodes.toctree)):
+ newnodes = []
+ includefiles = map(str, toctreenode['includefiles'])
+ indent += ' '
+ for includefile in includefiles:
+ if includefile not in traversed:
+ try:
+ traversed.append(includefile)
+ logger.info(indent + colorfunc(includefile))
+ subtree = inline_all_toctrees(builder, docnameset, includefile,
+ builder.env.get_doctree(includefile),
+ colorfunc, traversed, indent)
+ docnameset.add(includefile)
+ except Exception:
+ logger.warning(__('toctree contains ref to nonexisting file %r'),
+ includefile, location=docname)
+ else:
+ sof = addnodes.start_of_file(docname=includefile)
+ sof.children = subtree.children
+ for sectionnode in sof.findall(nodes.section):
+ if 'docname' not in sectionnode:
+ sectionnode['docname'] = includefile
+ newnodes.append(sof)
+ toctreenode.parent.replace(toctreenode, newnodes)
+ return tree
+
+
+def _make_id(string: str) -> str:
"""Convert `string` into an identifier and return it.
This function is a modified version of ``docutils.nodes.make_id()`` of
@@ -157,75 +464,218 @@ def _make_id(string: str) ->str:
# Maintainer: docutils-develop@lists.sourceforge.net
# Copyright: This module has been placed in the public domain.
"""
- pass
+ id = string.translate(_non_id_translate_digraphs)
+ id = id.translate(_non_id_translate)
+ # get rid of non-ascii characters.
+ # 'ascii' lowercase to prevent problems with turkish locale.
+ id = unicodedata.normalize('NFKD', id).encode('ascii', 'ignore').decode('ascii')
+ # shrink runs of whitespace and replace by hyphen
+ id = _non_id_chars.sub('-', ' '.join(id.split()))
+ id = _non_id_at_ends.sub('', id)
+ return str(id)
_non_id_chars = re.compile('[^a-zA-Z0-9._]+')
_non_id_at_ends = re.compile('^[-0-9._]+|-+$')
-_non_id_translate = {(248): 'o', (273): 'd', (295): 'h', (305): 'i', (322):
- 'l', (359): 't', (384): 'b', (387): 'b', (392): 'c', (396): 'd', (402):
- 'f', (409): 'k', (410): 'l', (414): 'n', (421): 'p', (427): 't', (429):
- 't', (436): 'y', (438): 'z', (485): 'g', (549): 'z', (564): 'l', (565):
- 'n', (566): 't', (567): 'j', (572): 'c', (575): 's', (576): 'z', (583):
- 'e', (585): 'j', (587): 'q', (589): 'r', (591): 'y'}
-_non_id_translate_digraphs = {(223): 'sz', (230): 'ae', (339): 'oe', (568):
- 'db', (569): 'qp'}
-
-
-def make_id(env: BuildEnvironment, document: nodes.document, prefix: str='',
- term: (str | None)=None) ->str:
+_non_id_translate = {
+ 0x00f8: 'o', # o with stroke
+ 0x0111: 'd', # d with stroke
+ 0x0127: 'h', # h with stroke
+ 0x0131: 'i', # dotless i
+ 0x0142: 'l', # l with stroke
+ 0x0167: 't', # t with stroke
+ 0x0180: 'b', # b with stroke
+ 0x0183: 'b', # b with topbar
+ 0x0188: 'c', # c with hook
+ 0x018c: 'd', # d with topbar
+ 0x0192: 'f', # f with hook
+ 0x0199: 'k', # k with hook
+ 0x019a: 'l', # l with bar
+ 0x019e: 'n', # n with long right leg
+ 0x01a5: 'p', # p with hook
+ 0x01ab: 't', # t with palatal hook
+ 0x01ad: 't', # t with hook
+ 0x01b4: 'y', # y with hook
+ 0x01b6: 'z', # z with stroke
+ 0x01e5: 'g', # g with stroke
+ 0x0225: 'z', # z with hook
+ 0x0234: 'l', # l with curl
+ 0x0235: 'n', # n with curl
+ 0x0236: 't', # t with curl
+ 0x0237: 'j', # dotless j
+ 0x023c: 'c', # c with stroke
+ 0x023f: 's', # s with swash tail
+ 0x0240: 'z', # z with swash tail
+ 0x0247: 'e', # e with stroke
+ 0x0249: 'j', # j with stroke
+ 0x024b: 'q', # q with hook tail
+ 0x024d: 'r', # r with stroke
+ 0x024f: 'y', # y with stroke
+}
+_non_id_translate_digraphs = {
+ 0x00df: 'sz', # ligature sz
+ 0x00e6: 'ae', # ae
+ 0x0153: 'oe', # ligature oe
+ 0x0238: 'db', # db digraph
+ 0x0239: 'qp', # qp digraph
+}
+
+
+def make_id(env: BuildEnvironment, document: nodes.document,
+ prefix: str = '', term: str | None = None) -> str:
"""Generate an appropriate node_id for given *prefix* and *term*."""
- pass
-
-
-def find_pending_xref_condition(node: addnodes.pending_xref, condition: str
- ) ->(Element | None):
+ node_id = None
+ if prefix:
+ idformat = prefix + "-%s"
+ else:
+ idformat = (document.settings.id_prefix or "id") + "%s"
+
+ # try to generate node_id by *term*
+ if prefix and term:
+ node_id = _make_id(idformat % term)
+ if node_id == prefix:
+ # *term* is not good to generate a node_id.
+ node_id = None
+ elif term:
+ node_id = _make_id(term)
+ if node_id == '':
+ node_id = None # fallback to None
+
+ while node_id is None or node_id in document.ids:
+ node_id = idformat % env.new_serialno(prefix)
+
+ return node_id
+
+
+def find_pending_xref_condition(node: addnodes.pending_xref, condition: str,
+ ) -> Element | None:
"""Pick matched pending_xref_condition node up from the pending_xref."""
- pass
+ for subnode in node:
+ if (isinstance(subnode, addnodes.pending_xref_condition) and
+ subnode.get('condition') == condition):
+ return subnode
+ return None
-def make_refnode(builder: Builder, fromdocname: str, todocname: str,
- targetid: (str | None), child: (Node | list[Node]), title: (str | None)
- =None) ->nodes.reference:
+def make_refnode(builder: Builder, fromdocname: str, todocname: str, targetid: str | None,
+ child: Node | list[Node], title: str | None = None,
+ ) -> nodes.reference:
"""Shortcut to create a reference node."""
- pass
+ node = nodes.reference('', '', internal=True)
+ if fromdocname == todocname and targetid:
+ node['refid'] = targetid
+ else:
+ if targetid:
+ node['refuri'] = (builder.get_relative_uri(fromdocname, todocname) +
+ '#' + targetid)
+ else:
+ node['refuri'] = builder.get_relative_uri(fromdocname, todocname)
+ if title:
+ node['reftitle'] = title
+ node += child
+ return node
-NON_SMARTQUOTABLE_PARENT_NODES = (nodes.FixedTextElement, nodes.literal,
- nodes.math, nodes.image, nodes.raw, nodes.problematic, addnodes.
- not_smartquotable)
+def set_source_info(directive: Directive, node: Node) -> None:
+ node.source, node.line = \
+ directive.state_machine.get_source_and_line(directive.lineno)
-def is_smartquotable(node: Node) ->bool:
- """Check whether the node is smart-quotable or not."""
- pass
+def set_role_source_info(inliner: Inliner, lineno: int, node: Node) -> None:
+ gsal = inliner.reporter.get_source_and_line # type: ignore[attr-defined]
+ node.source, node.line = gsal(lineno)
-def process_only_nodes(document: Node, tags: Tags) ->None:
- """Filter ``only`` nodes which do not match *tags*."""
- pass
+def copy_source_info(src: Element, dst: Element) -> None:
+ with contextlib.suppress(ValueError):
+ dst.source = get_node_source(src)
+ dst.line = get_node_line(src)
-def _only_node_keep_children(node: addnodes.only, tags: Tags) ->bool:
+NON_SMARTQUOTABLE_PARENT_NODES = (
+ nodes.FixedTextElement,
+ nodes.literal,
+ nodes.math,
+ nodes.image,
+ nodes.raw,
+ nodes.problematic,
+ addnodes.not_smartquotable,
+)
+
+
+def is_smartquotable(node: Node) -> bool:
+ """Check whether the node is smart-quotable or not."""
+ for pnode in traverse_parent(node.parent):
+ if isinstance(pnode, NON_SMARTQUOTABLE_PARENT_NODES):
+ return False
+ if pnode.get('support_smartquotes', None) is False:
+ return False
+
+ return getattr(node, 'support_smartquotes', None) is not False
+
+
+def process_only_nodes(document: Node, tags: Tags) -> None:
+ """Filter ``only`` nodes which do not match *tags*."""
+ for node in document.findall(addnodes.only):
+ if _only_node_keep_children(node, tags):
+ node.replace_self(node.children or nodes.comment())
+ else:
+ # A comment on the comment() nodes being inserted: replacing by [] would
+ # result in a "Losing ids" exception if there is a target node before
+ # the only node, so we make sure docutils can transfer the id to
+ # something, even if it's just a comment and will lose the id anyway...
+ node.replace_self(nodes.comment())
+
+
+def _only_node_keep_children(node: addnodes.only, tags: Tags) -> bool:
"""Keep children if tags match or error."""
- pass
+ try:
+ return tags.eval_condition(node['expr'])
+ except Exception as err:
+ logger.warning(
+ __('exception while evaluating only directive expression: %s'),
+ err,
+ location=node)
+ return True
-def _copy_except__document(el: Element) ->Element:
+def _copy_except__document(el: Element) -> Element:
"""Monkey-patch ```nodes.Element.copy``` to not copy the ``_document``
attribute.
xref: https://github.com/sphinx-doc/sphinx/issues/11116#issuecomment-1376767086
"""
- pass
+ newnode = object.__new__(el.__class__)
+ # set in Element.__init__()
+ newnode.children = []
+ newnode.rawsource = el.rawsource
+ newnode.tagname = el.tagname
+ # copied in Element.copy()
+ newnode.attributes = {k: (v
+ if k not in {'ids', 'classes', 'names', 'dupnames', 'backrefs'}
+ else v[:])
+ for k, v in el.attributes.items()}
+ newnode.line = el.line
+ newnode.source = el.source
+ return newnode
-nodes.Element.copy = _copy_except__document
+nodes.Element.copy = _copy_except__document # type: ignore[assignment]
-def _deepcopy(el: Element) ->Element:
+def _deepcopy(el: Element) -> Element:
"""Monkey-patch ```nodes.Element.deepcopy``` for speed."""
- pass
-
-
-nodes.Element.deepcopy = _deepcopy
+ newnode = el.copy()
+ newnode.children = [child.deepcopy() for child in el.children]
+ for child in newnode.children:
+ child.parent = newnode
+ if el.document:
+ child.document = el.document
+ if child.source is None:
+ child.source = el.document.current_source
+ if child.line is None:
+ child.line = el.document.current_line
+ return newnode
+
+
+nodes.Element.deepcopy = _deepcopy # type: ignore[assignment]
diff --git a/sphinx/util/osutil.py b/sphinx/util/osutil.py
index 83ada565b..d5d6ab5bc 100644
--- a/sphinx/util/osutil.py
+++ b/sphinx/util/osutil.py
@@ -1,5 +1,7 @@
"""Operating system-related utility functions for Sphinx."""
+
from __future__ import annotations
+
import contextlib
import filecmp
import os
@@ -11,34 +13,65 @@ from io import StringIO
from os import path
from pathlib import Path
from typing import TYPE_CHECKING
+
from sphinx.locale import __
+
if TYPE_CHECKING:
from types import TracebackType
from typing import Any
-SEP = '/'
+
+# SEP separates path elements in the canonical file names
+#
+# Define SEP as a manifest constant, not so much because we expect it to change
+# in the future as to avoid the suspicion that a stray "/" in the code is a
+# hangover from more *nix-oriented origins.
+SEP = "/"
+
+
+def os_path(canonical_path: str, /) -> str:
+ return canonical_path.replace(SEP, path.sep)
-def canon_path(native_path: (str | os.PathLike[str]), /) ->str:
+def canon_path(native_path: str | os.PathLike[str], /) -> str:
"""Return path in OS-independent form"""
- pass
+ return os.fspath(native_path).replace(path.sep, SEP)
-def path_stabilize(filepath: (str | os.PathLike[str]), /) ->str:
+def path_stabilize(filepath: str | os.PathLike[str], /) -> str:
"""Normalize path separator and unicode string"""
- pass
+ new_path = canon_path(filepath)
+ return unicodedata.normalize('NFC', new_path)
-def relative_uri(base: str, to: str) ->str:
+def relative_uri(base: str, to: str) -> str:
"""Return a relative URL from ``base`` to ``to``."""
- pass
-
-
-def ensuredir(file: (str | os.PathLike[str])) ->None:
+ if to.startswith(SEP):
+ return to
+ b2 = base.split('#')[0].split(SEP)
+ t2 = to.split('#')[0].split(SEP)
+ # remove common segments (except the last segment)
+ for x, y in zip(b2[:-1], t2[:-1], strict=False):
+ if x != y:
+ break
+ b2.pop(0)
+ t2.pop(0)
+ if b2 == t2:
+ # Special case: relative_uri('f/index.html','f/index.html')
+ # returns '', not 'index.html'
+ return ''
+ if len(b2) == 1 and t2 == ['']:
+ # Special case: relative_uri('f/index.html','f/') should
+ # return './', not ''
+ return '.' + SEP
+ return ('..' + SEP) * (len(b2) - 1) + SEP.join(t2)
+
+
+def ensuredir(file: str | os.PathLike[str]) -> None:
"""Ensure that a path exists."""
- pass
+ os.makedirs(file, exist_ok=True)
-def _last_modified_time(source: (str | os.PathLike[str]), /) ->int:
+def _last_modified_time(source: str | os.PathLike[str], /) -> int:
"""Return the last modified time of ``filename``.
The time is returned as integer microseconds.
@@ -48,17 +81,23 @@ def _last_modified_time(source: (str | os.PathLike[str]), /) ->int:
We prefer to err on the side of re-rendering a file,
so we round up to the nearest microsecond.
"""
- pass
+ st = source.stat() if isinstance(source, os.DirEntry) else os.stat(source)
+ # upside-down floor division to get the ceiling
+ return -(st.st_mtime_ns // -1_000)
-def _copy_times(source: (str | os.PathLike[str]), dest: (str | os.PathLike[
- str])) ->None:
+def _copy_times(source: str | os.PathLike[str], dest: str | os.PathLike[str]) -> None:
"""Copy a file's modification times."""
- pass
+ st = source.stat() if isinstance(source, os.DirEntry) else os.stat(source)
+ os.utime(dest, ns=(st.st_atime_ns, st.st_mtime_ns))
-def copyfile(source: (str | os.PathLike[str]), dest: (str | os.PathLike[str
- ]), *, force: bool=False) ->None:
+def copyfile(
+ source: str | os.PathLike[str],
+ dest: str | os.PathLike[str],
+ *,
+ force: bool = False,
+) -> None:
"""Copy a file and its modification times, if possible.
:param source: An existing source to copy.
@@ -68,41 +107,87 @@ def copyfile(source: (str | os.PathLike[str]), dest: (str | os.PathLike[str
.. note:: :func:`copyfile` is a no-op if *source* and *dest* are identical.
"""
- pass
+ # coerce to Path objects
+ source = Path(source)
+ dest = Path(dest)
+ if not source.exists():
+ msg = f'{source} does not exist'
+ raise FileNotFoundError(msg)
+
+ if (
+ not (dest_exists := dest.exists()) or
+ # comparison must be done using shallow=False since
+ # two different files might have the same size
+ not filecmp.cmp(source, dest, shallow=False)
+ ):
+ if not force and dest_exists:
+ # sphinx.util.logging imports sphinx.util.osutil,
+ # so use a local import to avoid circular imports
+ from sphinx.util import logging
+ logger = logging.getLogger(__name__)
+
+ msg = __('Aborted attempted copy from %s to %s '
+ '(the destination path has existing data).')
+ logger.warning(msg, source, dest,
+ type='misc', subtype='copy_overwrite')
+ return
+
+ shutil.copyfile(source, dest)
+ with contextlib.suppress(OSError):
+ # don't do full copystat because the source may be read-only
+ _copy_times(source, dest)
+
+
+_no_fn_re = re.compile(r'[^a-zA-Z0-9_-]')
-_no_fn_re = re.compile('[^a-zA-Z0-9_-]')
+def make_filename(string: str) -> str:
+ return _no_fn_re.sub('', string) or 'sphinx'
-def relpath(path: (str | os.PathLike[str]), start: (str | os.PathLike[str] |
- None)=os.curdir) ->str:
+def make_filename_from_project(project: str) -> str:
+ return make_filename(project.removesuffix(' Documentation')).lower()
+
+
+def relpath(path: str | os.PathLike[str],
+ start: str | os.PathLike[str] | None = os.curdir) -> str:
"""Return a relative filepath to *path* either from the current directory or
from an optional *start* directory.
This is an alternative of ``os.path.relpath()``. This returns original path
if *path* and *start* are on different drives (for Windows platform).
"""
- pass
+ try:
+ return os.path.relpath(path, start)
+ except ValueError:
+ return str(path)
-safe_relpath = relpath
+safe_relpath = relpath # for compatibility
fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
+
+
abspath = path.abspath
class _chdir:
"""Remove this fall-back once support for Python 3.10 is removed."""
- def __init__(self, target_dir: str, /) ->None:
+ def __init__(self, target_dir: str, /) -> None:
self.path = target_dir
self._dirs: list[str] = []
- def __enter__(self) ->None:
+ def __enter__(self) -> None:
self._dirs.append(os.getcwd())
os.chdir(self.path)
- def __exit__(self, type: (type[BaseException] | None), value: (
- BaseException | None), traceback: (TracebackType | None), /) ->None:
+ def __exit__(
+ self,
+ type: type[BaseException] | None,
+ value: BaseException | None,
+ traceback: TracebackType | None,
+ /,
+ ) -> None:
os.chdir(self._dirs.pop())
@@ -123,26 +208,55 @@ class FileAvoidWrite:
Objects can be used as context managers.
"""
- def __init__(self, path: (str | Path)) ->None:
+ def __init__(self, path: str | Path) -> None:
self._path = path
self._io: StringIO | None = None
- def close(self) ->None:
+ def write(self, data: str) -> None:
+ if not self._io:
+ self._io = StringIO()
+ self._io.write(data)
+
+ def close(self) -> None:
"""Stop accepting writes and write file, if needed."""
- pass
+ if not self._io:
+ msg = 'FileAvoidWrite does not support empty files.'
+ raise Exception(msg)
+
+ buf = self.getvalue()
+ self._io.close()
+
+ try:
+ with open(self._path, encoding='utf-8') as old_f:
+ old_content = old_f.read()
+ if old_content == buf:
+ return
+ except OSError:
+ pass
+
+ with open(self._path, 'w', encoding='utf-8') as f:
+ f.write(buf)
- def __enter__(self) ->FileAvoidWrite:
+ def __enter__(self) -> FileAvoidWrite:
return self
- def __exit__(self, exc_type: type[Exception], exc_value: Exception,
- traceback: Any) ->bool:
+ def __exit__(
+ self, exc_type: type[Exception], exc_value: Exception, traceback: Any,
+ ) -> bool:
self.close()
return True
- def __getattr__(self, name: str) ->Any:
+ def __getattr__(self, name: str) -> Any:
+ # Proxy to _io instance.
if not self._io:
- msg = (
- 'Must write to FileAvoidWrite before other methods can be used'
- )
+ msg = 'Must write to FileAvoidWrite before other methods can be used'
raise Exception(msg)
+
return getattr(self._io, name)
+
+
+def rmtree(path: str) -> None:
+ if os.path.isdir(path):
+ shutil.rmtree(path)
+ else:
+ os.remove(path)
diff --git a/sphinx/util/parallel.py b/sphinx/util/parallel.py
index d4b1bfccb..f17ef7129 100644
--- a/sphinx/util/parallel.py
+++ b/sphinx/util/parallel.py
@@ -1,39 +1,159 @@
"""Parallel building utilities."""
+
from __future__ import annotations
+
import os
import time
import traceback
from math import sqrt
from typing import TYPE_CHECKING, Any
+
try:
import multiprocessing
HAS_MULTIPROCESSING = True
except ImportError:
HAS_MULTIPROCESSING = False
+
from sphinx.errors import SphinxParallelError
from sphinx.util import logging
+
if TYPE_CHECKING:
from collections.abc import Callable, Sequence
+
logger = logging.getLogger(__name__)
+
+# our parallel functionality only works for the forking Process
parallel_available = HAS_MULTIPROCESSING and os.name == 'posix'
class SerialTasks:
"""Has the same interface as ParallelTasks, but executes tasks directly."""
- def __init__(self, nproc: int=1) ->None:
+ def __init__(self, nproc: int = 1) -> None:
+ pass
+
+ def add_task(
+ self, task_func: Callable, arg: Any = None, result_func: Callable | None = None,
+ ) -> None:
+ if arg is not None:
+ res = task_func(arg)
+ else:
+ res = task_func()
+ if result_func:
+ result_func(res)
+
+ def join(self) -> None:
pass
class ParallelTasks:
"""Executes *nproc* tasks in parallel after forking."""
- def __init__(self, nproc: int) ->None:
+ def __init__(self, nproc: int) -> None:
self.nproc = nproc
+ # (optional) function performed by each task on the result of main task
self._result_funcs: dict[int, Callable] = {}
+ # task arguments
self._args: dict[int, list[Any] | None] = {}
+ # list of subprocesses (both started and waiting)
self._procs: dict[int, Any] = {}
+ # list of receiving pipe connections of running subprocesses
self._precvs: dict[int, Any] = {}
+ # list of receiving pipe connections of waiting subprocesses
self._precvsWaiting: dict[int, Any] = {}
+ # number of working subprocesses
self._pworking = 0
+ # task number of each subprocess
self._taskid = 0
+
+ def _process(self, pipe: Any, func: Callable, arg: Any) -> None:
+ try:
+ collector = logging.LogCollector()
+ with collector.collect():
+ if arg is None:
+ ret = func()
+ else:
+ ret = func(arg)
+ failed = False
+ except BaseException as err:
+ failed = True
+ errmsg = traceback.format_exception_only(err.__class__, err)[0].strip()
+ ret = (errmsg, traceback.format_exc())
+ logging.convert_serializable(collector.logs)
+ pipe.send((failed, collector.logs, ret))
+
+ def add_task(
+ self, task_func: Callable, arg: Any = None, result_func: Callable | None = None,
+ ) -> None:
+ tid = self._taskid
+ self._taskid += 1
+ self._result_funcs[tid] = result_func or (lambda arg, result: None)
+ self._args[tid] = arg
+ precv, psend = multiprocessing.Pipe(False)
+ context: Any = multiprocessing.get_context('fork')
+ proc = context.Process(target=self._process, args=(psend, task_func, arg))
+ self._procs[tid] = proc
+ self._precvsWaiting[tid] = precv
+ try:
+ self._join_one()
+ except Exception:
+ # shutdown other child processes on failure
+ # (e.g. OSError: Failed to allocate memory)
+ self.terminate()
+
+ def join(self) -> None:
+ try:
+ while self._pworking:
+ if not self._join_one():
+ time.sleep(0.02)
+ finally:
+ # shutdown other child processes on failure
+ self.terminate()
+
+ def terminate(self) -> None:
+ for tid in list(self._precvs):
+ self._procs[tid].terminate()
+ self._result_funcs.pop(tid)
+ self._procs.pop(tid)
+ self._precvs.pop(tid)
+ self._pworking -= 1
+
+ def _join_one(self) -> bool:
+ joined_any = False
+ for tid, pipe in self._precvs.items():
+ if pipe.poll():
+ exc, logs, result = pipe.recv()
+ if exc:
+ raise SphinxParallelError(*result)
+ for log in logs:
+ logger.handle(log)
+ self._result_funcs.pop(tid)(self._args.pop(tid), result)
+ self._procs[tid].join()
+ self._precvs.pop(tid)
+ self._pworking -= 1
+ joined_any = True
+ break
+
+ while self._precvsWaiting and self._pworking < self.nproc:
+ newtid, newprecv = self._precvsWaiting.popitem()
+ self._precvs[newtid] = newprecv
+ self._procs[newtid].start()
+ self._pworking += 1
+
+ return joined_any
+
+
+def make_chunks(arguments: Sequence[str], nproc: int, maxbatch: int = 10) -> list[Any]:
+ # determine how many documents to read in one go
+ nargs = len(arguments)
+ chunksize = nargs // nproc
+ if chunksize >= maxbatch:
+ # try to improve batch size vs. number of batches
+ chunksize = int(sqrt(nargs / nproc * maxbatch))
+ if chunksize == 0:
+ chunksize = 1
+ nchunks, rest = divmod(nargs, chunksize)
+ if rest:
+ nchunks += 1
+ # partition documents in "chunks" that will be written by one Process
+ return [arguments[i * chunksize:(i + 1) * chunksize] for i in range(nchunks)]
diff --git a/sphinx/util/parsing.py b/sphinx/util/parsing.py
index cc99e270e..a8f937f8f 100644
--- a/sphinx/util/parsing.py
+++ b/sphinx/util/parsing.py
@@ -1,17 +1,28 @@
"""Docutils utility functions for parsing text."""
+
from __future__ import annotations
+
import contextlib
from typing import TYPE_CHECKING
+
from docutils.nodes import Element, Node
from docutils.statemachine import StringList, string2lines
+
if TYPE_CHECKING:
from collections.abc import Iterator
+
from docutils.parsers.rst.states import RSTState
-def nested_parse_to_nodes(state: RSTState, text: (str | StringList), *,
- source: str='<generated text>', offset: int=0, allow_section_headings:
- bool=True, keep_title_context: bool=False) ->list[Node]:
+def nested_parse_to_nodes(
+ state: RSTState,
+ text: str | StringList,
+ *,
+ source: str = '<generated text>',
+ offset: int = 0,
+ allow_section_headings: bool = True,
+ keep_title_context: bool = False,
+) -> list[Node]: # Element | nodes.Text
"""Parse *text* into nodes.
:param state:
@@ -40,4 +51,43 @@ def nested_parse_to_nodes(state: RSTState, text: (str | StringList), *,
.. versionadded:: 7.4
"""
- pass
+ document = state.document
+ content = _text_to_string_list(
+ text, source=source, tab_width=document.settings.tab_width,
+ )
+ node = Element() # Anonymous container for parsing
+ node.document = document
+
+ if keep_title_context:
+ state.nested_parse(content, offset, node, match_titles=allow_section_headings)
+ else:
+ with _fresh_title_style_context(state):
+ state.nested_parse(content, offset, node, match_titles=allow_section_headings)
+ return node.children
+
+
+@contextlib.contextmanager
+def _fresh_title_style_context(state: RSTState) -> Iterator[None]:
+ # hack around title style bookkeeping
+ memo = state.memo
+ surrounding_title_styles: list[str | tuple[str, str]] = memo.title_styles
+ surrounding_section_level: int = memo.section_level
+ # clear current title styles
+ memo.title_styles = []
+ memo.section_level = 0
+ try:
+ yield
+ finally:
+ # reset title styles
+ memo.title_styles = surrounding_title_styles
+ memo.section_level = surrounding_section_level
+
+
+def _text_to_string_list(
+ text: str | StringList, /, *, source: str, tab_width: int,
+) -> StringList:
+ # Doesn't really belong in this module, but avoids circular imports.
+ if isinstance(text, StringList):
+ return text
+ content = string2lines(text, tab_width, convert_whitespace=True)
+ return StringList(content, source=source)
diff --git a/sphinx/util/png.py b/sphinx/util/png.py
index fb20e2105..6c942194e 100644
--- a/sphinx/util/png.py
+++ b/sphinx/util/png.py
@@ -1,22 +1,43 @@
"""PNG image manipulation helpers."""
+
from __future__ import annotations
+
import binascii
import struct
+
LEN_IEND = 12
LEN_DEPTH = 22
+
DEPTH_CHUNK_LEN = struct.pack('!i', 10)
DEPTH_CHUNK_START = b'tEXtDepth\x00'
-IEND_CHUNK = b'\x00\x00\x00\x00IEND\xaeB`\x82'
+IEND_CHUNK = b'\x00\x00\x00\x00IEND\xAE\x42\x60\x82'
-def read_png_depth(filename: str) ->(int | None):
+def read_png_depth(filename: str) -> int | None:
"""Read the special tEXt chunk indicating the depth from a PNG file."""
- pass
+ with open(filename, 'rb') as f:
+ f.seek(- (LEN_IEND + LEN_DEPTH), 2)
+ depthchunk = f.read(LEN_DEPTH)
+ if not depthchunk.startswith(DEPTH_CHUNK_LEN + DEPTH_CHUNK_START):
+ # either not a PNG file or not containing the depth chunk
+ return None
+ else:
+ return struct.unpack('!i', depthchunk[14:18])[0]
-def write_png_depth(filename: str, depth: int) ->None:
+def write_png_depth(filename: str, depth: int) -> None:
"""Write the special tEXt chunk indicating the depth to a PNG file.
The chunk is placed immediately before the special IEND chunk.
"""
- pass
+ data = struct.pack('!i', depth)
+ with open(filename, 'r+b') as f:
+ # seek to the beginning of the IEND chunk
+ f.seek(-LEN_IEND, 2)
+ # overwrite it with the depth chunk
+ f.write(DEPTH_CHUNK_LEN + DEPTH_CHUNK_START + data)
+ # calculate the checksum over chunk name and data
+ crc = binascii.crc32(DEPTH_CHUNK_START + data) & 0xffffffff
+ f.write(struct.pack('!I', crc))
+ # replace the IEND chunk
+ f.write(IEND_CHUNK)
diff --git a/sphinx/util/requests.py b/sphinx/util/requests.py
index c1ee76baa..7c64e940d 100644
--- a/sphinx/util/requests.py
+++ b/sphinx/util/requests.py
@@ -1,45 +1,76 @@
"""Simple requests package loader"""
+
from __future__ import annotations
+
import warnings
from typing import Any
from urllib.parse import urlsplit
+
import requests
from urllib3.exceptions import InsecureRequestWarning
+
import sphinx
-_USER_AGENT = (
- f'Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0 Sphinx/{sphinx.__version__}'
- )
+
+_USER_AGENT = (f'Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0 '
+ f'Sphinx/{sphinx.__version__}')
-def _get_tls_cacert(url: str, certs: (str | dict[str, str] | None)) ->(str |
- bool):
+def _get_tls_cacert(url: str, certs: str | dict[str, str] | None) -> str | bool:
"""Get additional CA cert for a specific URL."""
- pass
+ if not certs:
+ return True
+ elif isinstance(certs, str | tuple):
+ return certs
+ else:
+ hostname = urlsplit(url).netloc
+ if '@' in hostname:
+ _, hostname = hostname.split('@', 1)
+ return certs.get(hostname, True)
-def get(url: str, **kwargs: Any) ->requests.Response:
+
+def get(url: str, **kwargs: Any) -> requests.Response:
"""Sends a GET request like ``requests.get()``.
This sets up User-Agent header and TLS verification automatically.
"""
- pass
+ with _Session() as session:
+ return session.get(url, **kwargs)
-def head(url: str, **kwargs: Any) ->requests.Response:
+def head(url: str, **kwargs: Any) -> requests.Response:
"""Sends a HEAD request like ``requests.head()``.
This sets up User-Agent header and TLS verification automatically.
"""
- pass
+ with _Session() as session:
+ return session.head(url, **kwargs)
class _Session(requests.Session):
-
- def request(self, method: str, url: str, _user_agent: str='', _tls_info:
- tuple[bool, str | dict[str, str] | None]=(), **kwargs: Any
- ) ->requests.Response:
+ def request( # type: ignore[override]
+ self, method: str, url: str,
+ _user_agent: str = '',
+ _tls_info: tuple[bool, str | dict[str, str] | None] = (), # type: ignore[assignment]
+ **kwargs: Any,
+ ) -> requests.Response:
"""Sends a request with an HTTP verb and url.
This sets up User-Agent header and TLS verification automatically.
"""
- pass
+ headers = kwargs.setdefault('headers', {})
+ headers.setdefault('User-Agent', _user_agent or _USER_AGENT)
+ if _tls_info:
+ tls_verify, tls_cacerts = _tls_info
+ verify = bool(kwargs.get('verify', tls_verify))
+ kwargs.setdefault('verify', verify and _get_tls_cacert(url, tls_cacerts))
+ else:
+ verify = kwargs.get('verify', True)
+
+ if verify:
+ return super().request(method, url, **kwargs)
+
+ with warnings.catch_warnings():
+ # ignore InsecureRequestWarning if verify=False
+ warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+ return super().request(method, url, **kwargs)
diff --git a/sphinx/util/rst.py b/sphinx/util/rst.py
index 21fe03367..4e8fdee2e 100644
--- a/sphinx/util/rst.py
+++ b/sphinx/util/rst.py
@@ -1,44 +1,112 @@
"""reST helper functions."""
+
from __future__ import annotations
+
import re
from collections import defaultdict
from contextlib import contextmanager
from typing import TYPE_CHECKING, cast
from unicodedata import east_asian_width
+
from docutils.parsers.rst import roles
-from docutils.parsers.rst.languages import en as english
+from docutils.parsers.rst.languages import en as english # type: ignore[attr-defined]
from docutils.parsers.rst.states import Body
from docutils.utils import Reporter
from jinja2 import Environment, pass_environment
+
from sphinx.locale import __
from sphinx.util import docutils, logging
+
if TYPE_CHECKING:
from collections.abc import Iterator
+
from docutils.statemachine import StringList
+
logger = logging.getLogger(__name__)
+
FIELD_NAME_RE = re.compile(Body.patterns['field_marker'])
-symbols_re = re.compile('([!-\\-/:-@\\[-`{-~])')
+symbols_re = re.compile(r'([!-\-/:-@\[-`{-~])') # symbols without dot(0x2e)
SECTIONING_CHARS = ['=', '-', '~']
-WIDECHARS: dict[str, str] = defaultdict(lambda : 'WF')
-WIDECHARS['ja'] = 'WFA'
+# width of characters
+WIDECHARS: dict[str, str] = defaultdict(lambda: "WF") # WF: Wide + Full-width
+WIDECHARS["ja"] = "WFA" # In Japanese, Ambiguous characters also have double width
+
+
+def escape(text: str) -> str:
+ text = symbols_re.sub(r'\\\1', text)
+ text = re.sub(r'^\.', r'\.', text) # escape a dot at top
+ return text
-def textwidth(text: str, widechars: str='WF') ->int:
+
+def textwidth(text: str, widechars: str = 'WF') -> int:
"""Get width of text."""
- pass
+ def charwidth(char: str, widechars: str) -> int:
+ if east_asian_width(char) in widechars:
+ return 2
+ else:
+ return 1
+
+ return sum(charwidth(c, widechars) for c in text)
@pass_environment
-def heading(env: Environment, text: str, level: int=1) ->str:
+def heading(env: Environment, text: str, level: int = 1) -> str:
"""Create a heading for *level*."""
- pass
+ assert level <= 3
+ # ``env.language`` is injected by ``sphinx.util.template.ReSTRenderer``
+ width = textwidth(text, WIDECHARS[env.language]) # type: ignore[attr-defined]
+ sectioning_char = SECTIONING_CHARS[level - 1]
+ return f'{text}\n{sectioning_char * width}'
+
+
+@contextmanager
+def default_role(docname: str, name: str) -> Iterator[None]:
+ if name:
+ dummy_reporter = Reporter('', 4, 4)
+ role_fn, _ = roles.role(name, english, 0, dummy_reporter)
+ if role_fn:
+ docutils.register_role('', role_fn) # type: ignore[arg-type]
+ else:
+ logger.warning(__('default role %s not found'), name, location=docname)
+
+ yield
+ docutils.unregister_role('')
-def prepend_prolog(content: StringList, prolog: str) ->None:
+
+def prepend_prolog(content: StringList, prolog: str) -> None:
"""Prepend a string to content body as prolog."""
- pass
+ if prolog:
+ pos = 0
+ for line in content:
+ if FIELD_NAME_RE.match(line):
+ pos += 1
+ else:
+ break
+
+ if pos > 0:
+ # insert a blank line after docinfo
+ content.insert(pos, '', '<generated>', 0)
+ pos += 1
+
+ # insert prolog (after docinfo if exists)
+ lineno = 0
+ for lineno, line in enumerate(prolog.splitlines()):
+ content.insert(pos + lineno, line, '<rst_prolog>', lineno)
+
+ content.insert(pos + lineno + 1, '', '<generated>', 0)
-def append_epilog(content: StringList, epilog: str) ->None:
+def append_epilog(content: StringList, epilog: str) -> None:
"""Append a string to content body as epilog."""
- pass
+ if epilog:
+ if len(content) > 0:
+ source, lineno = content.info(-1)
+ lineno = cast(int, lineno) # lineno will never be None, since len(content) > 0
+ else:
+ source = '<generated>'
+ lineno = 0
+ content.append('', source, lineno + 1)
+ for lineno, line in enumerate(epilog.splitlines()):
+ content.append(line, '<rst_epilog>', lineno)
diff --git a/sphinx/util/tags.py b/sphinx/util/tags.py
index 808df44b6..71492dc9a 100644
--- a/sphinx/util/tags.py
+++ b/sphinx/util/tags.py
@@ -1,43 +1,112 @@
from __future__ import annotations
+
import warnings
from typing import TYPE_CHECKING
+
import jinja2.environment
import jinja2.nodes
import jinja2.parser
+
from sphinx.deprecation import RemovedInSphinx90Warning
+
if TYPE_CHECKING:
from collections.abc import Iterator, Sequence
from typing import Literal
+
_ENV = jinja2.environment.Environment()
class BooleanParser(jinja2.parser.Parser):
"""Only allow conditional expressions and binary operators."""
+ def parse_compare(self) -> jinja2.nodes.Expr:
+ node: jinja2.nodes.Expr
+ token = self.stream.current
+ if token.type == 'name':
+ if token.value in {'true', 'True'}:
+ node = jinja2.nodes.Const(True, lineno=token.lineno)
+ elif token.value in {'false', 'False'}:
+ node = jinja2.nodes.Const(False, lineno=token.lineno)
+ elif token.value in {'none', 'None'}:
+ node = jinja2.nodes.Const(None, lineno=token.lineno)
+ else:
+ node = jinja2.nodes.Name(token.value, 'load', lineno=token.lineno)
+ next(self.stream)
+ elif token.type == 'lparen':
+ next(self.stream)
+ node = self.parse_expression()
+ self.stream.expect('rparen')
+ else:
+ self.fail(f"unexpected token '{token}'", token.lineno)
+ return node
-class Tags:
- def __init__(self, tags: Sequence[str]=()) ->None:
+class Tags:
+ def __init__(self, tags: Sequence[str] = ()) -> None:
self._tags = set(tags or ())
self._condition_cache: dict[str, bool] = {}
- def __str__(self) ->str:
- return f"{self.__class__.__name__}({', '.join(sorted(self._tags))})"
+ def __str__(self) -> str:
+ return f'{self.__class__.__name__}({", ".join(sorted(self._tags))})'
- def __repr__(self) ->str:
+ def __repr__(self) -> str:
return f'{self.__class__.__name__}({tuple(sorted(self._tags))})'
- def __iter__(self) ->Iterator[str]:
+ def __iter__(self) -> Iterator[str]:
return iter(self._tags)
- def __contains__(self, tag: str) ->bool:
+ def __contains__(self, tag: str) -> bool:
return tag in self._tags
- def eval_condition(self, condition: str) ->bool:
+ def has(self, tag: str) -> bool:
+ return tag in self._tags
+
+ def add(self, tag: str) -> None:
+ self._tags.add(tag)
+
+ def remove(self, tag: str) -> None:
+ self._tags.discard(tag)
+
+ @property
+ def tags(self) -> dict[str, Literal[True]]:
+ warnings.warn('Tags.tags is deprecated, use methods on Tags.',
+ RemovedInSphinx90Warning, stacklevel=2)
+ return dict.fromkeys(self._tags, True)
+
+ def eval_condition(self, condition: str) -> bool:
"""Evaluate a boolean condition.
Only conditional expressions and binary operators (and, or, not)
are permitted, and operate on tag names, where truthy values mean
the tag is present and vice versa.
"""
- pass
+ if condition in self._condition_cache:
+ return self._condition_cache[condition]
+
+ # exceptions are handled by the caller
+ parser = BooleanParser(_ENV, condition, state='variable')
+ expr = parser.parse_expression()
+ if not parser.stream.eos:
+ msg = 'chunk after expression'
+ raise ValueError(msg)
+
+ evaluated = self._condition_cache[condition] = self._eval_node(expr)
+ return evaluated
+
+ def _eval_node(self, node: jinja2.nodes.Node | None) -> bool:
+ if isinstance(node, jinja2.nodes.CondExpr):
+ if self._eval_node(node.test):
+ return self._eval_node(node.expr1)
+ else:
+ return self._eval_node(node.expr2)
+ elif isinstance(node, jinja2.nodes.And):
+ return self._eval_node(node.left) and self._eval_node(node.right)
+ elif isinstance(node, jinja2.nodes.Or):
+ return self._eval_node(node.left) or self._eval_node(node.right)
+ elif isinstance(node, jinja2.nodes.Not):
+ return not self._eval_node(node.node)
+ elif isinstance(node, jinja2.nodes.Name):
+ return node.name in self._tags
+ else:
+ msg = 'invalid node, check parsing'
+ raise ValueError(msg)
diff --git a/sphinx/util/template.py b/sphinx/util/template.py
index 5598ce1aa..2b38e3a86 100644
--- a/sphinx/util/template.py
+++ b/sphinx/util/template.py
@@ -1,61 +1,89 @@
"""Templates utility functions for Sphinx."""
+
from __future__ import annotations
+
import os
from functools import partial
from os import path
from typing import TYPE_CHECKING, Any
+
from jinja2 import TemplateNotFound
from jinja2.loaders import BaseLoader
from jinja2.sandbox import SandboxedEnvironment
+
from sphinx import package_dir
from sphinx.jinja2glue import SphinxFileSystemLoader
from sphinx.locale import get_translator
from sphinx.util import rst, texescape
+
if TYPE_CHECKING:
from collections.abc import Callable, Sequence
+
from jinja2.environment import Environment
class BaseRenderer:
-
- def __init__(self, loader: (BaseLoader | None)=None) ->None:
- self.env = SandboxedEnvironment(loader=loader, extensions=[
- 'jinja2.ext.i18n'])
+ def __init__(self, loader: BaseLoader | None = None) -> None:
+ self.env = SandboxedEnvironment(loader=loader, extensions=['jinja2.ext.i18n'])
self.env.filters['repr'] = repr
- self.env.install_gettext_translations(get_translator())
+ # ``install_gettext_translations`` is injected by the ``jinja2.ext.i18n`` extension
+ self.env.install_gettext_translations(get_translator()) # type: ignore[attr-defined]
+ def render(self, template_name: str, context: dict[str, Any]) -> str:
+ return self.env.get_template(template_name).render(context)
+
+ def render_string(self, source: str, context: dict[str, Any]) -> str:
+ return self.env.from_string(source).render(context)
-class FileRenderer(BaseRenderer):
- def __init__(self, search_path: Sequence[str | os.PathLike[str]]) ->None:
+class FileRenderer(BaseRenderer):
+ def __init__(self, search_path: Sequence[str | os.PathLike[str]]) -> None:
if isinstance(search_path, str | os.PathLike):
search_path = [search_path]
else:
+ # filter "None" paths
search_path = list(filter(None, search_path))
+
loader = SphinxFileSystemLoader(search_path)
super().__init__(loader)
+ @classmethod
+ def render_from_file(
+ cls: type[FileRenderer], filename: str, context: dict[str, Any],
+ ) -> str:
+ dirname = os.path.dirname(filename)
+ basename = os.path.basename(filename)
+ return cls(dirname).render(basename, context)
-class SphinxRenderer(FileRenderer):
- def __init__(self, template_path: (Sequence[str | os.PathLike[str]] |
- None)=None) ->None:
+class SphinxRenderer(FileRenderer):
+ def __init__(self, template_path: Sequence[str | os.PathLike[str]] | None = None) -> None:
if template_path is None:
template_path = os.path.join(package_dir, 'templates')
super().__init__(template_path)
+ @classmethod
+ def render_from_file(
+ cls: type[FileRenderer], filename: str, context: dict[str, Any],
+ ) -> str:
+ return FileRenderer.render_from_file(filename, context)
-class LaTeXRenderer(SphinxRenderer):
- def __init__(self, template_path: (Sequence[str | os.PathLike[str]] |
- None)=None, latex_engine: (str | None)=None) ->None:
+class LaTeXRenderer(SphinxRenderer):
+ def __init__(self, template_path: Sequence[str | os.PathLike[str]] | None = None,
+ latex_engine: str | None = None) -> None:
if template_path is None:
template_path = [os.path.join(package_dir, 'templates', 'latex')]
super().__init__(template_path)
+
+ # use texescape as escape filter
escape = partial(texescape.escape, latex_engine=latex_engine)
self.env.filters['e'] = escape
self.env.filters['escape'] = escape
self.env.filters['eabbr'] = texescape.escape_abbr
+
+ # use JSP/eRuby like tagging instead because curly bracket; the default
+ # tagging of jinja2 is not good for LaTeX sources.
self.env.variable_start_string = '<%='
self.env.variable_end_string = '%>'
self.env.block_start_string = '<%'
@@ -65,11 +93,14 @@ class LaTeXRenderer(SphinxRenderer):
class ReSTRenderer(SphinxRenderer):
-
- def __init__(self, template_path: (Sequence[str | os.PathLike[str]] |
- None)=None, language: (str | None)=None) ->None:
+ def __init__(self, template_path: Sequence[str | os.PathLike[str]] | None = None,
+ language: str | None = None) -> None:
super().__init__(template_path)
+
+ # add language to environment
self.env.extend(language=language)
+
+ # use texescape as escape filter
self.env.filters['e'] = rst.escape
self.env.filters['escape'] = rst.escape
self.env.filters['heading'] = rst.heading
@@ -78,15 +109,32 @@ class ReSTRenderer(SphinxRenderer):
class SphinxTemplateLoader(BaseLoader):
"""A loader supporting template inheritance"""
- def __init__(self, confdir: (str | os.PathLike[str]), templates_paths:
- Sequence[str | os.PathLike[str]], system_templates_paths: Sequence[
- str | os.PathLike[str]]) ->None:
+ def __init__(self, confdir: str | os.PathLike[str],
+ templates_paths: Sequence[str | os.PathLike[str]],
+ system_templates_paths: Sequence[str | os.PathLike[str]]) -> None:
self.loaders = []
self.sysloaders = []
+
for templates_path in templates_paths:
loader = SphinxFileSystemLoader(path.join(confdir, templates_path))
self.loaders.append(loader)
+
for templates_path in system_templates_paths:
loader = SphinxFileSystemLoader(templates_path)
self.loaders.append(loader)
self.sysloaders.append(loader)
+
+ def get_source(self, environment: Environment, template: str) -> tuple[str, str, Callable]:
+ if template.startswith('!'):
+ # search a template from ``system_templates_paths``
+ loaders = self.sysloaders
+ template = template[1:]
+ else:
+ loaders = self.loaders
+
+ for loader in loaders:
+ try:
+ return loader.get_source(environment, template)
+ except TemplateNotFound:
+ pass
+ raise TemplateNotFound(template)
diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py
index 2ed1eb943..8527441d7 100644
--- a/sphinx/util/texescape.py
+++ b/sphinx/util/texescape.py
@@ -1,47 +1,153 @@
"""TeX escaping helper."""
+
from __future__ import annotations
+
import re
-tex_replacements = [('$', '\\$'), ('%', '\\%'), ('&', '\\&'), ('#', '\\#'),
- ('_', '\\_'), ('{', '\\{'), ('}', '\\}'), ('\\', '\\textbackslash{}'),
- ('~', '\\textasciitilde{}'), ('^', '\\textasciicircum{}'), ('[', '{[}'),
- (']', '{]}'), ('✓', '\\(\\checkmark\\)'), ('✔',
- '\\(\\pmb{\\checkmark}\\)'), ('✕', '\\(\\times\\)'), ('✖',
- '\\(\\pmb{\\times}\\)'), ('\ufeff', '{}'), ('⎽', '\\_'), ('ℯ', 'e'), (
- 'ⅈ', 'i')]
-ascii_tex_replacements = [('-', '\\sphinxhyphen{}'), ("'",
- '\\textquotesingle{}'), ('`', '\\textasciigrave{}'), ('<',
- '\\textless{}'), ('>', '\\textgreater{}')]
-unicode_tex_replacements = [('¶', '\\P{}'), ('§', '\\S{}'), ('€',
- '\\texteuro{}'), ('∞', '\\(\\infty\\)'), ('±', '\\(\\pm\\)'), ('→',
- '\\(\\rightarrow\\)'), ('‣', '\\(\\rightarrow\\)'), ('–',
- '\\textendash{}'), ('⁰', '\\(\\sp{\\text{0}}\\)'), ('¹',
- '\\(\\sp{\\text{1}}\\)'), ('²', '\\(\\sp{\\text{2}}\\)'), ('³',
- '\\(\\sp{\\text{3}}\\)'), ('⁴', '\\(\\sp{\\text{4}}\\)'), ('⁵',
- '\\(\\sp{\\text{5}}\\)'), ('⁶', '\\(\\sp{\\text{6}}\\)'), ('⁷',
- '\\(\\sp{\\text{7}}\\)'), ('⁸', '\\(\\sp{\\text{8}}\\)'), ('⁹',
- '\\(\\sp{\\text{9}}\\)'), ('₀', '\\(\\sb{\\text{0}}\\)'), ('₁',
- '\\(\\sb{\\text{1}}\\)'), ('₂', '\\(\\sb{\\text{2}}\\)'), ('₃',
- '\\(\\sb{\\text{3}}\\)'), ('₄', '\\(\\sb{\\text{4}}\\)'), ('₅',
- '\\(\\sb{\\text{5}}\\)'), ('₆', '\\(\\sb{\\text{6}}\\)'), ('₇',
- '\\(\\sb{\\text{7}}\\)'), ('₈', '\\(\\sb{\\text{8}}\\)'), ('₉',
- '\\(\\sb{\\text{9}}\\)')]
+
+tex_replacements = [
+ # map TeX special chars
+ ('$', r'\$'),
+ ('%', r'\%'),
+ ('&', r'\&'),
+ ('#', r'\#'),
+ ('_', r'\_'),
+ ('{', r'\{'),
+ ('}', r'\}'),
+ ('\\', r'\textbackslash{}'),
+ ('~', r'\textasciitilde{}'),
+ ('^', r'\textasciicircum{}'),
+ # map chars to avoid mis-interpretation in LaTeX
+ ('[', r'{[}'),
+ (']', r'{]}'),
+ # map special Unicode characters to TeX commands
+ ('✓', r'\(\checkmark\)'),
+ ('✔', r'\(\pmb{\checkmark}\)'),
+ ('✕', r'\(\times\)'),
+ ('✖', r'\(\pmb{\times}\)'),
+ # used to separate -- in options
+ ('', r'{}'),
+ # map some special Unicode characters to similar ASCII ones
+ # (even for Unicode LaTeX as may not be supported by OpenType font)
+ ('⎽', r'\_'),
+ ('ℯ', r'e'),
+ ('ⅈ', r'i'),
+ # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc
+ # OHM SIGN U+2126 is handled by LaTeX textcomp package
+]
+
+# A map to avoid TeX ligatures or character replacements in PDF output
+# xelatex/lualatex/uplatex are handled differently (#5790, #6888)
+ascii_tex_replacements = [
+ # Note: the " renders curly in OT1 encoding but straight in T1, T2A, LY1...
+ # escaping it to \textquotedbl would break documents using OT1
+ # Sphinx does \shorthandoff{"} to avoid problems with some languages
+ # There is no \text... LaTeX escape for the hyphen character -
+ ('-', r'\sphinxhyphen{}'), # -- and --- are TeX ligatures
+ # ,, is a TeX ligature in T1 encoding, but escaping the comma adds
+ # complications (whether by {}, or a macro) and is not done
+ # the next two require textcomp package
+ ("'", r'\textquotesingle{}'), # else ' renders curly, and '' is a ligature
+ ('`', r'\textasciigrave{}'), # else \` and \`\` render curly
+ ('<', r'\textless{}'), # < is inv. exclam in OT1, << is a T1-ligature
+ ('>', r'\textgreater{}'), # > is inv. quest. mark in 0T1, >> a T1-ligature
+]
+
+# A map Unicode characters to LaTeX representation
+# (for LaTeX engines which don't support unicode)
+unicode_tex_replacements = [
+ # map some more common Unicode characters to TeX commands
+ ('¶', r'\P{}'),
+ ('§', r'\S{}'),
+ ('€', r'\texteuro{}'),
+ ('∞', r'\(\infty\)'),
+ ('±', r'\(\pm\)'),
+ ('→', r'\(\rightarrow\)'),
+ ('‣', r'\(\rightarrow\)'),
+ ('–', r'\textendash{}'),
+ # superscript
+ ('⁰', r'\(\sp{\text{0}}\)'),
+ ('¹', r'\(\sp{\text{1}}\)'),
+ ('²', r'\(\sp{\text{2}}\)'),
+ ('³', r'\(\sp{\text{3}}\)'),
+ ('⁴', r'\(\sp{\text{4}}\)'),
+ ('⁵', r'\(\sp{\text{5}}\)'),
+ ('⁶', r'\(\sp{\text{6}}\)'),
+ ('⁷', r'\(\sp{\text{7}}\)'),
+ ('⁸', r'\(\sp{\text{8}}\)'),
+ ('⁹', r'\(\sp{\text{9}}\)'),
+ # subscript
+ ('₀', r'\(\sb{\text{0}}\)'),
+ ('₁', r'\(\sb{\text{1}}\)'),
+ ('₂', r'\(\sb{\text{2}}\)'),
+ ('₃', r'\(\sb{\text{3}}\)'),
+ ('₄', r'\(\sb{\text{4}}\)'),
+ ('₅', r'\(\sb{\text{5}}\)'),
+ ('₆', r'\(\sb{\text{6}}\)'),
+ ('₇', r'\(\sb{\text{7}}\)'),
+ ('₈', r'\(\sb{\text{8}}\)'),
+ ('₉', r'\(\sb{\text{9}}\)'),
+]
+
+# TODO: this should be called tex_idescape_map because its only use is in
+# sphinx.writers.latex.LaTeXTranslator.idescape()
+# %, {, }, \, #, and ~ are the only ones which must be replaced by _ character
+# It would be simpler to define it entirely here rather than in init().
+# Unicode replacements are superfluous, as idescape() uses backslashreplace
tex_replace_map: dict[int, str] = {}
+
_tex_escape_map: dict[int, str] = {}
_tex_escape_map_without_unicode: dict[int, str] = {}
_tex_hlescape_map: dict[int, str] = {}
_tex_hlescape_map_without_unicode: dict[int, str] = {}
-def escape(s: str, latex_engine: (str | None)=None) ->str:
+def escape(s: str, latex_engine: str | None = None) -> str:
"""Escape text for LaTeX output."""
- pass
+ if latex_engine in ('lualatex', 'xelatex'):
+ # unicode based LaTeX engine
+ return s.translate(_tex_escape_map_without_unicode)
+ else:
+ return s.translate(_tex_escape_map)
-def hlescape(s: str, latex_engine: (str | None)=None) ->str:
+def hlescape(s: str, latex_engine: str | None = None) -> str:
"""Escape text for LaTeX highlighter."""
- pass
+ if latex_engine in ('lualatex', 'xelatex'):
+ # unicode based LaTeX engine
+ return s.translate(_tex_hlescape_map_without_unicode)
+ else:
+ return s.translate(_tex_hlescape_map)
-def escape_abbr(text: str) ->str:
+def escape_abbr(text: str) -> str:
"""Adjust spacing after abbreviations. Works with @ letter or other."""
- pass
+ return re.sub(r'\.(?=\s|$)', r'.\@{}', text)
+
+
+def init() -> None:
+ for a, b in tex_replacements:
+ _tex_escape_map[ord(a)] = b
+ _tex_escape_map_without_unicode[ord(a)] = b
+ tex_replace_map[ord(a)] = '_'
+
+ # no reason to do this for _tex_escape_map_without_unicode
+ for a, b in ascii_tex_replacements:
+ _tex_escape_map[ord(a)] = b
+
+ # but the hyphen has a specific PDF bookmark problem
+ # https://github.com/latex3/hyperref/issues/112
+ _tex_escape_map_without_unicode[ord('-')] = r'\sphinxhyphen{}'
+
+ for a, b in unicode_tex_replacements:
+ _tex_escape_map[ord(a)] = b
+ # This is actually unneeded:
+ tex_replace_map[ord(a)] = '_'
+
+ for a, b in tex_replacements:
+ if a in '[]{}\\':
+ continue
+ _tex_hlescape_map[ord(a)] = b
+ _tex_hlescape_map_without_unicode[ord(a)] = b
+
+ for a, b in unicode_tex_replacements:
+ _tex_hlescape_map[ord(a)] = b
diff --git a/sphinx/util/typing.py b/sphinx/util/typing.py
index e0151d793..dbad5457c 100644
--- a/sphinx/util/typing.py
+++ b/sphinx/util/typing.py
@@ -1,5 +1,7 @@
"""The composite types for Sphinx."""
+
from __future__ import annotations
+
import dataclasses
import sys
import types
@@ -7,71 +9,130 @@ import typing
from collections.abc import Callable, Sequence
from contextvars import Context, ContextVar, Token
from struct import Struct
-from typing import TYPE_CHECKING, Annotated, Any, ForwardRef, NewType, TypedDict, TypeVar, Union
+from typing import (
+ TYPE_CHECKING,
+ Annotated,
+ Any,
+ ForwardRef,
+ NewType,
+ TypedDict,
+ TypeVar,
+ Union,
+)
+
from docutils import nodes
from docutils.parsers.rst.states import Inliner
+
from sphinx.util import logging
+
if TYPE_CHECKING:
from collections.abc import Mapping
from typing import Final, Literal, Protocol, TypeAlias
+
from typing_extensions import TypeIs
+
from sphinx.application import Sphinx
- _RestifyMode: TypeAlias = Literal['fully-qualified-except-typing', 'smart']
- _StringifyMode: TypeAlias = Literal['fully-qualified-except-typing',
- 'fully-qualified', 'smart']
+
+ _RestifyMode: TypeAlias = Literal[
+ 'fully-qualified-except-typing',
+ 'smart',
+ ]
+ _StringifyMode: TypeAlias = Literal[
+ 'fully-qualified-except-typing',
+ 'fully-qualified',
+ 'smart',
+ ]
+
logger = logging.getLogger(__name__)
-_INVALID_BUILTIN_CLASSES: Final[Mapping[object, str]] = {Context:
- 'contextvars.Context', ContextVar: 'contextvars.ContextVar', Token:
- 'contextvars.Token', Struct: 'struct.Struct', types.AsyncGeneratorType:
- 'types.AsyncGeneratorType', types.BuiltinFunctionType:
- 'types.BuiltinFunctionType', types.BuiltinMethodType:
- 'types.BuiltinMethodType', types.CellType: 'types.CellType', types.
- ClassMethodDescriptorType: 'types.ClassMethodDescriptorType', types.
- CodeType: 'types.CodeType', types.CoroutineType: 'types.CoroutineType',
- types.FrameType: 'types.FrameType', types.FunctionType:
- 'types.FunctionType', types.GeneratorType: 'types.GeneratorType', types
- .GetSetDescriptorType: 'types.GetSetDescriptorType', types.LambdaType:
- 'types.LambdaType', types.MappingProxyType: 'types.MappingProxyType',
- types.MemberDescriptorType: 'types.MemberDescriptorType', types.
- MethodDescriptorType: 'types.MethodDescriptorType', types.MethodType:
- 'types.MethodType', types.MethodWrapperType: 'types.MethodWrapperType',
- types.ModuleType: 'types.ModuleType', types.TracebackType:
- 'types.TracebackType', types.WrapperDescriptorType:
- 'types.WrapperDescriptorType'}
-
-
-def is_invalid_builtin_class(obj: Any) ->bool:
+
+
+# classes that have an incorrect .__module__ attribute
+_INVALID_BUILTIN_CLASSES: Final[Mapping[object, str]] = {
+ Context: 'contextvars.Context', # Context.__module__ == '_contextvars'
+ ContextVar: 'contextvars.ContextVar', # ContextVar.__module__ == '_contextvars'
+ Token: 'contextvars.Token', # Token.__module__ == '_contextvars'
+ Struct: 'struct.Struct', # Struct.__module__ == '_struct'
+ # types in 'types' with <type>.__module__ == 'builtins':
+ types.AsyncGeneratorType: 'types.AsyncGeneratorType',
+ types.BuiltinFunctionType: 'types.BuiltinFunctionType',
+ types.BuiltinMethodType: 'types.BuiltinMethodType',
+ types.CellType: 'types.CellType',
+ types.ClassMethodDescriptorType: 'types.ClassMethodDescriptorType',
+ types.CodeType: 'types.CodeType',
+ types.CoroutineType: 'types.CoroutineType',
+ types.FrameType: 'types.FrameType',
+ types.FunctionType: 'types.FunctionType',
+ types.GeneratorType: 'types.GeneratorType',
+ types.GetSetDescriptorType: 'types.GetSetDescriptorType',
+ types.LambdaType: 'types.LambdaType',
+ types.MappingProxyType: 'types.MappingProxyType',
+ types.MemberDescriptorType: 'types.MemberDescriptorType',
+ types.MethodDescriptorType: 'types.MethodDescriptorType',
+ types.MethodType: 'types.MethodType',
+ types.MethodWrapperType: 'types.MethodWrapperType',
+ types.ModuleType: 'types.ModuleType',
+ types.TracebackType: 'types.TracebackType',
+ types.WrapperDescriptorType: 'types.WrapperDescriptorType',
+}
+
+
+def is_invalid_builtin_class(obj: Any) -> bool:
"""Check *obj* is an invalid built-in class."""
- pass
+ try:
+ return obj in _INVALID_BUILTIN_CLASSES
+ except TypeError: # unhashable type
+ return False
+# Text like nodes which are initialized with text and rawsource
TextlikeNode: TypeAlias = nodes.Text | nodes.TextElement
-PathMatcher: TypeAlias = Callable[[str], bool]
-if TYPE_CHECKING:
+# path matcher
+PathMatcher: TypeAlias = Callable[[str], bool]
+# common role functions
+if TYPE_CHECKING:
class RoleFunction(Protocol):
-
- def __call__(self, name: str, rawtext: str, text: str, lineno: int,
- inliner: Inliner, /, options: (dict[str, Any] | None)=None,
- content: Sequence[str]=()) ->tuple[list[nodes.Node], list[nodes
- .system_message]]:
+ def __call__(
+ self,
+ name: str,
+ rawtext: str,
+ text: str,
+ lineno: int,
+ inliner: Inliner,
+ /,
+ options: dict[str, Any] | None = None,
+ content: Sequence[str] = (),
+ ) -> tuple[list[nodes.Node], list[nodes.system_message]]:
...
else:
- RoleFunction: TypeAlias = Callable[[str, str, str, int, Inliner, dict[
- str, Any], Sequence[str]], tuple[list[nodes.Node], list[nodes.
- system_message]]]
+ RoleFunction: TypeAlias = Callable[
+ [str, str, str, int, Inliner, dict[str, Any], Sequence[str]],
+ tuple[list[nodes.Node], list[nodes.system_message]],
+ ]
+
+# A option spec for directive
OptionSpec: TypeAlias = dict[str, Callable[[str], Any]]
+
+# title getter functions for enumerable nodes (see sphinx.domains.std)
TitleGetter: TypeAlias = Callable[[nodes.Node], str]
-InventoryItem: TypeAlias = tuple[str, str, str, str]
+
+# inventory data on memory
+InventoryItem: TypeAlias = tuple[
+ str, # project name
+ str, # project version
+ str, # URL
+ str, # display name
+]
Inventory: TypeAlias = dict[str, dict[str, InventoryItem]]
-class ExtensionMetadata(TypedDict, total=(False)):
+class ExtensionMetadata(TypedDict, total=False):
"""The metadata returned by an extension's ``setup()`` function.
See :ref:`ext-metadata`.
"""
+
version: str
"""The extension version (default: ``'unknown version'``)."""
env_version: int
@@ -90,35 +151,65 @@ if TYPE_CHECKING:
_ExtensionSetupFunc: TypeAlias = Callable[[Sphinx], ExtensionMetadata]
-def get_type_hints(obj: Any, globalns: (dict[str, Any] | None)=None,
- localns: (dict[str, Any] | None)=None, include_extras: bool=False) ->dict[
- str, Any]:
+def get_type_hints(
+ obj: Any,
+ globalns: dict[str, Any] | None = None,
+ localns: dict[str, Any] | None = None,
+ include_extras: bool = False,
+) -> dict[str, Any]:
"""Return a dictionary containing type hints for a function, method, module or class
object.
This is a simple wrapper of `typing.get_type_hints()` that does not raise an error on
runtime.
"""
- pass
-
-
-def is_system_TypeVar(typ: Any) ->bool:
+ from sphinx.util.inspect import safe_getattr # lazy loading
+
+ try:
+ return typing.get_type_hints(obj, globalns, localns, include_extras=include_extras)
+ except NameError:
+ # Failed to evaluate ForwardRef (maybe TYPE_CHECKING)
+ return safe_getattr(obj, '__annotations__', {})
+ except AttributeError:
+ # Failed to evaluate ForwardRef (maybe not runtime checkable)
+ return safe_getattr(obj, '__annotations__', {})
+ except TypeError:
+ # Invalid object is given. But try to get __annotations__ as a fallback.
+ return safe_getattr(obj, '__annotations__', {})
+ except KeyError:
+ # a broken class found (refs: https://github.com/sphinx-doc/sphinx/issues/8084)
+ return {}
+
+
+def is_system_TypeVar(typ: Any) -> bool:
"""Check *typ* is system defined TypeVar."""
- pass
+ modname = getattr(typ, '__module__', '')
+ return modname == 'typing' and isinstance(typ, TypeVar)
-def _is_annotated_form(obj: Any) ->TypeIs[Annotated[Any, ...]]:
+def _is_annotated_form(obj: Any) -> TypeIs[Annotated[Any, ...]]:
"""Check if *obj* is an annotated type."""
- pass
+ return typing.get_origin(obj) is Annotated or str(obj).startswith('typing.Annotated')
-def _is_unpack_form(obj: Any) ->bool:
+def _is_unpack_form(obj: Any) -> bool:
"""Check if the object is :class:`typing.Unpack` or equivalent."""
- pass
+ if sys.version_info >= (3, 11):
+ from typing import Unpack
+
+ # typing_extensions.Unpack != typing.Unpack for 3.11, but we assume
+ # that typing_extensions.Unpack should not be used in that case
+ return typing.get_origin(obj) is Unpack
+
+ # Python 3.10 requires typing_extensions.Unpack
+ origin = typing.get_origin(obj)
+ return (
+ getattr(origin, '__module__', None) == 'typing_extensions'
+ and origin.__name__ == 'Unpack'
+ )
-def restify(cls: Any, mode: _RestifyMode='fully-qualified-except-typing'
- ) ->str:
+def restify(cls: Any, mode: _RestifyMode = 'fully-qualified-except-typing') -> str:
"""Convert a type-like object to a reST reference.
:param mode: Specify a method how annotations will be stringified.
@@ -129,11 +220,149 @@ def restify(cls: Any, mode: _RestifyMode='fully-qualified-except-typing'
'smart'
Show the name of the annotation.
"""
- pass
-
-
-def stringify_annotation(annotation: Any, /, mode: _StringifyMode=
- 'fully-qualified-except-typing') ->str:
+ from sphinx.ext.autodoc.mock import ismock, ismockmodule # lazy loading
+ from sphinx.util.inspect import isgenericalias, object_description # lazy loading
+
+ valid_modes = {'fully-qualified-except-typing', 'smart'}
+ if mode not in valid_modes:
+ valid = ', '.join(map(repr, sorted(valid_modes)))
+ msg = f'mode must be one of {valid}; got {mode!r}'
+ raise ValueError(msg)
+
+ # things that are not types
+ if cls is None or cls == types.NoneType:
+ return ':py:obj:`None`'
+ if cls is Ellipsis:
+ return '...'
+ if isinstance(cls, str):
+ return cls
+
+ cls_module_is_typing = getattr(cls, '__module__', '') == 'typing'
+
+ # If the mode is 'smart', we always use '~'.
+ # If the mode is 'fully-qualified-except-typing',
+ # we use '~' only for the objects in the ``typing`` module.
+ module_prefix = '~' if mode == 'smart' or cls_module_is_typing else ''
+
+ try:
+ if ismockmodule(cls):
+ return f':py:class:`{module_prefix}{cls.__name__}`'
+ elif ismock(cls):
+ return f':py:class:`{module_prefix}{cls.__module__}.{cls.__name__}`'
+ elif is_invalid_builtin_class(cls):
+ # The above predicate never raises TypeError but should not be
+ # evaluated before determining whether *cls* is a mocked object
+ # or not; instead of two try-except blocks, we keep it here.
+ return f':py:class:`{module_prefix}{_INVALID_BUILTIN_CLASSES[cls]}`'
+ elif _is_annotated_form(cls):
+ args = restify(cls.__args__[0], mode)
+ meta_args = []
+ for m in cls.__metadata__:
+ if isinstance(m, type):
+ meta_args.append(restify(m, mode))
+ elif dataclasses.is_dataclass(m):
+ # use restify for the repr of field values rather than repr
+ d_fields = ', '.join([
+ fr"{f.name}=\ {restify(getattr(m, f.name), mode)}"
+ for f in dataclasses.fields(m) if f.repr
+ ])
+ meta_args.append(fr'{restify(type(m), mode)}\ ({d_fields})')
+ else:
+ meta_args.append(repr(m))
+ meta = ', '.join(meta_args)
+ if sys.version_info[:2] <= (3, 11):
+ # Hardcoded to fix errors on Python 3.11 and earlier.
+ return fr':py:class:`~typing.Annotated`\ [{args}, {meta}]'
+ return (f':py:class:`{module_prefix}{cls.__module__}.{cls.__name__}`'
+ fr'\ [{args}, {meta}]')
+ elif isinstance(cls, NewType):
+ return f':py:class:`{module_prefix}{cls.__module__}.{cls.__name__}`' # type: ignore[attr-defined]
+ elif isinstance(cls, types.UnionType):
+ # Union types (PEP 585) retain their definition order when they
+ # are printed natively and ``None``-like types are kept as is.
+ return ' | '.join(restify(a, mode) for a in cls.__args__)
+ elif cls.__module__ in ('__builtin__', 'builtins'):
+ if hasattr(cls, '__args__'):
+ if not cls.__args__: # Empty tuple, list, ...
+ return fr':py:class:`{cls.__name__}`\ [{cls.__args__!r}]'
+
+ concatenated_args = ', '.join(restify(arg, mode) for arg in cls.__args__)
+ return fr':py:class:`{cls.__name__}`\ [{concatenated_args}]'
+ return f':py:class:`{cls.__name__}`'
+ elif (isgenericalias(cls)
+ and cls_module_is_typing
+ and cls.__origin__ is Union):
+ # *cls* is defined in ``typing``, and thus ``__args__`` must exist
+ return ' | '.join(restify(a, mode) for a in cls.__args__)
+ elif isgenericalias(cls):
+ if isinstance(cls.__origin__, typing._SpecialForm):
+ # ClassVar; Concatenate; Final; Literal; Unpack; TypeGuard; TypeIs
+ # Required/NotRequired
+ text = restify(cls.__origin__, mode)
+ elif cls.__name__:
+ text = f':py:class:`{module_prefix}{cls.__module__}.{cls.__name__}`'
+ else:
+ text = restify(cls.__origin__, mode)
+
+ __args__ = getattr(cls, '__args__', ())
+ if not __args__:
+ return text
+ if all(map(is_system_TypeVar, __args__)):
+ # Don't print the arguments; they're all system defined type variables.
+ return text
+
+ # Callable has special formatting
+ if (
+ (cls_module_is_typing and cls.__name__ == 'Callable')
+ or (cls.__module__ == 'collections.abc' and cls.__name__ == 'Callable')
+ ):
+ args = ', '.join(restify(a, mode) for a in __args__[:-1])
+ returns = restify(__args__[-1], mode)
+ return fr'{text}\ [[{args}], {returns}]'
+
+ if cls_module_is_typing and cls.__origin__.__name__ == 'Literal':
+ args = ', '.join(_format_literal_arg_restify(a, mode=mode)
+ for a in cls.__args__)
+ return fr'{text}\ [{args}]'
+
+ # generic representation of the parameters
+ args = ', '.join(restify(a, mode) for a in __args__)
+ return fr'{text}\ [{args}]'
+ elif isinstance(cls, typing._SpecialForm):
+ return f':py:obj:`~{cls.__module__}.{cls.__name__}`' # type: ignore[attr-defined]
+ elif sys.version_info[:2] >= (3, 11) and cls is typing.Any:
+ # handle bpo-46998
+ return f':py:obj:`~{cls.__module__}.{cls.__name__}`'
+ elif hasattr(cls, '__qualname__'):
+ return f':py:class:`{module_prefix}{cls.__module__}.{cls.__qualname__}`'
+ elif isinstance(cls, ForwardRef):
+ return f':py:class:`{cls.__forward_arg__}`'
+ else:
+ # not a class (ex. TypeVar) but should have a __name__
+ return f':py:obj:`{module_prefix}{cls.__module__}.{cls.__name__}`'
+ except (AttributeError, TypeError) as exc:
+ logger.debug('restify on %r in mode %r failed: %r', cls, mode, exc)
+ return object_description(cls)
+
+
+def _format_literal_arg_restify(arg: Any, /, *, mode: str) -> str:
+ from sphinx.util.inspect import isenumattribute # lazy loading
+
+ if isenumattribute(arg):
+ enum_cls = arg.__class__
+ if mode == 'smart' or enum_cls.__module__ == 'typing':
+ # MyEnum.member
+ return f':py:attr:`~{enum_cls.__module__}.{enum_cls.__qualname__}.{arg.name}`'
+ # module.MyEnum.member
+ return f':py:attr:`{enum_cls.__module__}.{enum_cls.__qualname__}.{arg.name}`'
+ return repr(arg)
+
+
+def stringify_annotation(
+ annotation: Any,
+ /,
+ mode: _StringifyMode = 'fully-qualified-except-typing',
+) -> str:
"""Stringify type annotation object.
:param annotation: The annotation to stringified.
@@ -147,17 +376,185 @@ def stringify_annotation(annotation: Any, /, mode: _StringifyMode=
'fully-qualified'
Show the module name and qualified name of the annotation.
"""
- pass
-
-
-_DEPRECATED_OBJECTS: dict[str, tuple[Any, str, tuple[int, int]]] = {}
-
-
-def __getattr__(name: str) ->Any:
+ from sphinx.ext.autodoc.mock import ismock, ismockmodule # lazy loading
+
+ valid_modes = {'fully-qualified-except-typing', 'fully-qualified', 'smart'}
+ if mode not in valid_modes:
+ valid = ', '.join(map(repr, sorted(valid_modes)))
+ msg = f'mode must be one of {valid}; got {mode!r}'
+ raise ValueError(msg)
+
+ # things that are not types
+ if annotation is None or annotation == types.NoneType:
+ return 'None'
+ if annotation is Ellipsis:
+ return '...'
+ if isinstance(annotation, str):
+ if annotation.startswith("'") and annotation.endswith("'"):
+ # Might be a double Forward-ref'ed type. Go unquoting.
+ return annotation[1:-1]
+ return annotation
+ if not annotation:
+ return repr(annotation)
+
+ module_prefix = '~' if mode == 'smart' else ''
+
+ # The values below must be strings if the objects are well-formed.
+ annotation_qualname: str = getattr(annotation, '__qualname__', '')
+ annotation_module: str = getattr(annotation, '__module__', '')
+ annotation_name: str = getattr(annotation, '__name__', '')
+ annotation_module_is_typing = annotation_module == 'typing'
+
+ # Extract the annotation's base type by considering formattable cases
+ if isinstance(annotation, TypeVar) and not _is_unpack_form(annotation):
+ # typing_extensions.Unpack is incorrectly determined as a TypeVar
+ if annotation_module_is_typing and mode in {'fully-qualified-except-typing', 'smart'}:
+ return annotation_name
+ return module_prefix + f'{annotation_module}.{annotation_name}'
+ elif isinstance(annotation, NewType):
+ return module_prefix + f'{annotation_module}.{annotation_name}'
+ elif ismockmodule(annotation):
+ return module_prefix + annotation_name
+ elif ismock(annotation):
+ return module_prefix + f'{annotation_module}.{annotation_name}'
+ elif is_invalid_builtin_class(annotation):
+ return module_prefix + _INVALID_BUILTIN_CLASSES[annotation]
+ elif _is_annotated_form(annotation): # for py310+
+ pass
+ elif annotation_module == 'builtins' and annotation_qualname:
+ args = getattr(annotation, '__args__', None)
+ if args is None:
+ return annotation_qualname
+
+ # PEP 585 generic
+ if not args: # Empty tuple, list, ...
+ return repr(annotation)
+
+ concatenated_args = ', '.join(stringify_annotation(arg, mode) for arg in args)
+ return f'{annotation_qualname}[{concatenated_args}]'
+ else:
+ # add other special cases that can be directly formatted
+ pass
+
+ module_prefix = f'{annotation_module}.'
+ annotation_forward_arg: str | None = getattr(annotation, '__forward_arg__', None)
+ if annotation_qualname or (annotation_module_is_typing and not annotation_forward_arg):
+ if mode == 'smart':
+ module_prefix = f'~{module_prefix}'
+ if annotation_module_is_typing and mode == 'fully-qualified-except-typing':
+ module_prefix = ''
+ elif _is_unpack_form(annotation) and annotation_module == 'typing_extensions':
+ module_prefix = '~' if mode == 'smart' else ''
+ else:
+ module_prefix = ''
+
+ if annotation_module_is_typing:
+ if annotation_forward_arg:
+ # handle ForwardRefs
+ qualname = annotation_forward_arg
+ else:
+ if annotation_name:
+ qualname = annotation_name
+ elif annotation_qualname:
+ qualname = annotation_qualname
+ else:
+ # in this case, we know that the annotation is a member
+ # of ``typing`` and all of them define ``__origin__``
+ qualname = stringify_annotation(
+ annotation.__origin__, 'fully-qualified-except-typing',
+ ).replace('typing.', '') # ex. Union
+ elif annotation_qualname:
+ qualname = annotation_qualname
+ elif hasattr(annotation, '__origin__'):
+ # instantiated generic provided by a user
+ qualname = stringify_annotation(annotation.__origin__, mode)
+ elif isinstance(annotation, types.UnionType):
+ qualname = 'types.UnionType'
+ else:
+ # we weren't able to extract the base type, appending arguments would
+ # only make them appear twice
+ return repr(annotation)
+
+ # Process the generic arguments (if any).
+ # They must be a list or a tuple, otherwise they are considered 'broken'.
+ annotation_args = getattr(annotation, '__args__', ())
+ if annotation_args and isinstance(annotation_args, list | tuple):
+ if (
+ qualname in {'Union', 'types.UnionType'}
+ and all(getattr(a, '__origin__', ...) is typing.Literal for a in annotation_args)
+ ):
+ # special case to flatten a Union of Literals into a literal
+ flattened_args = typing.Literal[annotation_args].__args__ # type: ignore[attr-defined]
+ args = ', '.join(_format_literal_arg_stringify(a, mode=mode)
+ for a in flattened_args)
+ return f'{module_prefix}Literal[{args}]'
+ if qualname in {'Optional', 'Union', 'types.UnionType'}:
+ return ' | '.join(stringify_annotation(a, mode) for a in annotation_args)
+ elif qualname == 'Callable':
+ args = ', '.join(stringify_annotation(a, mode) for a in annotation_args[:-1])
+ returns = stringify_annotation(annotation_args[-1], mode)
+ return f'{module_prefix}Callable[[{args}], {returns}]'
+ elif qualname == 'Literal':
+ args = ', '.join(_format_literal_arg_stringify(a, mode=mode)
+ for a in annotation_args)
+ return f'{module_prefix}Literal[{args}]'
+ elif _is_annotated_form(annotation): # for py310+
+ args = stringify_annotation(annotation_args[0], mode)
+ meta_args = []
+ for m in annotation.__metadata__:
+ if isinstance(m, type):
+ meta_args.append(stringify_annotation(m, mode))
+ elif dataclasses.is_dataclass(m):
+ # use stringify_annotation for the repr of field values rather than repr
+ d_fields = ', '.join([
+ f"{f.name}={stringify_annotation(getattr(m, f.name), mode)}"
+ for f in dataclasses.fields(m) if f.repr
+ ])
+ meta_args.append(f'{stringify_annotation(type(m), mode)}({d_fields})')
+ else:
+ meta_args.append(repr(m))
+ meta = ', '.join(meta_args)
+ if sys.version_info[:2] <= (3, 11):
+ if mode == 'fully-qualified-except-typing':
+ return f'Annotated[{args}, {meta}]'
+ module_prefix = module_prefix.replace('builtins', 'typing')
+ return f'{module_prefix}Annotated[{args}, {meta}]'
+ return f'{module_prefix}Annotated[{args}, {meta}]'
+ elif all(is_system_TypeVar(a) for a in annotation_args):
+ # Suppress arguments if all system defined TypeVars (ex. Dict[KT, VT])
+ return module_prefix + qualname
+ else:
+ args = ', '.join(stringify_annotation(a, mode) for a in annotation_args)
+ return f'{module_prefix}{qualname}[{args}]'
+
+ return module_prefix + qualname
+
+
+def _format_literal_arg_stringify(arg: Any, /, *, mode: str) -> str:
+ from sphinx.util.inspect import isenumattribute # lazy loading
+
+ if isenumattribute(arg):
+ enum_cls = arg.__class__
+ if mode == 'smart' or enum_cls.__module__ == 'typing':
+ # MyEnum.member
+ return f'{enum_cls.__qualname__}.{arg.name}'
+ # module.MyEnum.member
+ return f'{enum_cls.__module__}.{enum_cls.__qualname__}.{arg.name}'
+ return repr(arg)
+
+
+# deprecated name -> (object to return, canonical path or empty string, removal version)
+_DEPRECATED_OBJECTS: dict[str, tuple[Any, str, tuple[int, int]]] = {
+}
+
+
+def __getattr__(name: str) -> Any:
if name not in _DEPRECATED_OBJECTS:
msg = f'module {__name__!r} has no attribute {name!r}'
raise AttributeError(msg)
+
from sphinx.deprecation import _deprecation_warning
+
deprecated_object, canonical_name, remove = _DEPRECATED_OBJECTS[name]
_deprecation_warning(__name__, name, canonical_name, remove=remove)
return deprecated_object
diff --git a/sphinx/versioning.py b/sphinx/versioning.py
index e75880d9e..506d7b575 100644
--- a/sphinx/versioning.py
+++ b/sphinx/versioning.py
@@ -1,27 +1,36 @@
"""Implements the low-level algorithms Sphinx uses for versioning doctrees."""
+
from __future__ import annotations
+
import pickle
from itertools import product, zip_longest
from operator import itemgetter
from os import path
from typing import TYPE_CHECKING, Any
from uuid import uuid4
+
from sphinx.transforms import SphinxTransform
+
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
+
from docutils.nodes import Node
+
from sphinx.application import Sphinx
from sphinx.util.typing import ExtensionMetadata
+
try:
- import Levenshtein
+ import Levenshtein # type: ignore[import-not-found]
+
IS_SPEEDUP = True
except ImportError:
IS_SPEEDUP = False
+
+# anything below that ratio is considered equal/changed
VERSIONING_RATIO = 65
-def add_uids(doctree: Node, condition: Callable[[Node], bool]) ->Iterator[Node
- ]:
+def add_uids(doctree: Node, condition: Callable[[Node], bool]) -> Iterator[Node]:
"""Add a unique id to every node in the `doctree` which matches the
condition and yield the nodes.
@@ -31,11 +40,14 @@ def add_uids(doctree: Node, condition: Callable[[Node], bool]) ->Iterator[Node
:param condition:
A callable which returns either ``True`` or ``False`` for a given node.
"""
- pass
+ for node in doctree.findall(condition):
+ node.uid = uuid4().hex # type: ignore[attr-defined]
+ yield node
-def merge_doctrees(old: Node, new: Node, condition: Callable[[Node], bool]
- ) ->Iterator[Node]:
+def merge_doctrees(
+ old: Node, new: Node, condition: Callable[[Node], bool]
+) -> Iterator[Node]:
"""Merge the `old` doctree with the `new` one while looking at nodes
matching the `condition`.
@@ -45,21 +57,128 @@ def merge_doctrees(old: Node, new: Node, condition: Callable[[Node], bool]
:param condition:
A callable which returns either ``True`` or ``False`` for a given node.
"""
- pass
+ old_iter = old.findall(condition)
+ new_iter = new.findall(condition)
+ old_nodes = []
+ new_nodes = []
+ ratios = {}
+ seen = set()
+ # compare the nodes each doctree in order
+ for old_node, new_node in zip_longest(old_iter, new_iter):
+ if old_node is None:
+ new_nodes.append(new_node)
+ continue
+ if not getattr(old_node, 'uid', None):
+ # maybe config.gettext_uuid has been changed.
+ old_node.uid = uuid4().hex # type: ignore[union-attr]
+ if new_node is None:
+ old_nodes.append(old_node)
+ continue
+ ratio = get_ratio(old_node.rawsource, new_node.rawsource) # type: ignore[union-attr]
+ if ratio == 0:
+ new_node.uid = old_node.uid # type: ignore[union-attr]
+ seen.add(new_node)
+ else:
+ ratios[old_node, new_node] = ratio
+ old_nodes.append(old_node)
+ new_nodes.append(new_node)
+ # calculate the ratios for each unequal pair of nodes, should we stumble
+ # on a pair which is equal we set the uid and add it to the seen ones
+ for old_node, new_node in product(old_nodes, new_nodes):
+ if new_node in seen or (old_node, new_node) in ratios:
+ continue
+ ratio = get_ratio(old_node.rawsource, new_node.rawsource) # type: ignore[union-attr]
+ if ratio == 0:
+ new_node.uid = old_node.uid # type: ignore[union-attr]
+ seen.add(new_node)
+ else:
+ ratios[old_node, new_node] = ratio
+ # choose the old node with the best ratio for each new node and set the uid
+ # as long as the ratio is under a certain value, in which case we consider
+ # them not changed but different
+ for (old_node, new_node), ratio in sorted(ratios.items(), key=itemgetter(1)):
+ if new_node in seen:
+ continue
+ else:
+ seen.add(new_node)
+ if ratio < VERSIONING_RATIO:
+ new_node.uid = old_node.uid # type: ignore[union-attr]
+ else:
+ new_node.uid = uuid4().hex # type: ignore[union-attr]
+ yield new_node
+ # create new uuids for any new node we left out earlier, this happens
+ # if one or more nodes are simply added.
+ for new_node in set(new_nodes) - seen:
+ new_node.uid = uuid4().hex # type: ignore[union-attr]
+ yield new_node
-def get_ratio(old: str, new: str) ->float:
+def get_ratio(old: str, new: str) -> float:
"""Return a "similarity ratio" (in percent) representing the similarity
between the two strings where 0 is equal and anything above less than equal.
"""
- pass
+ if not all([old, new]):
+ return VERSIONING_RATIO
+ if IS_SPEEDUP:
+ return Levenshtein.distance(old, new) / (len(old) / 100.0)
+ else:
+ return levenshtein_distance(old, new) / (len(old) / 100.0)
-def levenshtein_distance(a: str, b: str) ->int:
+
+def levenshtein_distance(a: str, b: str) -> int:
"""Return the Levenshtein edit distance between two strings *a* and *b*."""
- pass
+ if a == b:
+ return 0
+ if len(a) < len(b):
+ a, b = b, a
+ if not a:
+ return len(b)
+ previous_row = list(range(len(b) + 1))
+ for i, column1 in enumerate(a):
+ current_row = [i + 1]
+ for j, column2 in enumerate(b):
+ insertions = previous_row[j + 1] + 1
+ deletions = current_row[j] + 1
+ substitutions = previous_row[j] + (column1 != column2)
+ current_row.append(min(insertions, deletions, substitutions))
+ previous_row = current_row
+ return previous_row[-1]
class UIDTransform(SphinxTransform):
"""Add UIDs to doctree for versioning."""
+
default_priority = 880
+
+ def apply(self, **kwargs: Any) -> None:
+ env = self.env
+ old_doctree = None
+ versioning_condition = env.versioning_condition
+ if not versioning_condition:
+ return
+
+ if env.versioning_compare:
+ # get old doctree
+ try:
+ filename = path.join(env.doctreedir, env.docname + '.doctree')
+ with open(filename, 'rb') as f:
+ old_doctree = pickle.load(f)
+ except OSError:
+ pass
+
+ # add uids for versioning
+ if not env.versioning_compare or old_doctree is None:
+ list(add_uids(self.document, versioning_condition))
+ else:
+ list(merge_doctrees(old_doctree, self.document, versioning_condition))
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_transform(UIDTransform)
+
+ return {
+ 'version': 'builtin',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py
index 1c2f29067..242846128 100644
--- a/sphinx/writers/html.py
+++ b/sphinx/writers/html.py
@@ -1,18 +1,44 @@
"""docutils writers handling Sphinx' custom nodes."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, cast
+
from docutils.writers.html4css1 import Writer
+
from sphinx.util import logging
from sphinx.writers.html5 import HTML5Translator
+
if TYPE_CHECKING:
from sphinx.builders.html import StandaloneHTMLBuilder
+
+
logger = logging.getLogger(__name__)
HTMLTranslator = HTML5Translator
+# A good overview of the purpose behind these classes can be found here:
+# https://www.arnebrodowski.de/blog/write-your-own-restructuredtext-writer.html
+
-class HTMLWriter(Writer):
- settings_default_overrides = {'embed_stylesheet': False}
+class HTMLWriter(Writer): # type: ignore[misc]
- def __init__(self, builder: StandaloneHTMLBuilder) ->None:
+ # override embed-stylesheet default value to False.
+ settings_default_overrides = {"embed_stylesheet": False}
+
+ def __init__(self, builder: StandaloneHTMLBuilder) -> None:
super().__init__()
self.builder = builder
+
+ def translate(self) -> None:
+ # sadly, this is mostly copied from parent class
+ visitor = self.builder.create_translator(self.document, self.builder)
+ self.visitor = cast(HTML5Translator, visitor)
+ self.document.walkabout(visitor)
+ self.output = self.visitor.astext()
+ for attr in ('head_prefix', 'stylesheet', 'head', 'body_prefix',
+ 'body_pre_docinfo', 'docinfo', 'body', 'fragment',
+ 'body_suffix', 'meta', 'title', 'subtitle', 'header',
+ 'footer', 'html_prolog', 'html_head', 'html_title',
+ 'html_subtitle', 'html_body'):
+ setattr(self, attr, getattr(visitor, attr, None))
+ self.clean_meta = ''.join(self.visitor.meta[2:])
diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py
index 761abb855..2d6338d8c 100644
--- a/sphinx/writers/html5.py
+++ b/sphinx/writers/html5.py
@@ -1,41 +1,64 @@
"""Experimental docutils writers for HTML5 handling Sphinx's custom nodes."""
+
from __future__ import annotations
+
import os
import posixpath
import re
import urllib.parse
from collections.abc import Iterable
from typing import TYPE_CHECKING, cast
+
from docutils import nodes
from docutils.writers.html5_polyglot import HTMLTranslator as BaseTranslator
+
from sphinx import addnodes
from sphinx.locale import _, __, admonitionlabels
from sphinx.util import logging
from sphinx.util.docutils import SphinxTranslator
from sphinx.util.images import get_image_size
+
if TYPE_CHECKING:
from docutils.nodes import Element, Node, Text
+
from sphinx.builders import Builder
from sphinx.builders.html import StandaloneHTMLBuilder
+
+
logger = logging.getLogger(__name__)
+# A good overview of the purpose behind these classes can be found here:
+# https://www.arnebrodowski.de/blog/write-your-own-restructuredtext-writer.html
+
-def multiply_length(length: str, scale: int) ->str:
+def multiply_length(length: str, scale: int) -> str:
"""Multiply *length* (width or height) by *scale*."""
- pass
+ matched = re.match(r'^(\d*\.?\d*)\s*(\S*)$', length)
+ if not matched:
+ return length
+ if scale == 100:
+ return length
+ amount, unit = matched.groups()
+ result = float(amount) * scale / 100
+ return f"{int(result)}{unit}"
-class HTML5Translator(SphinxTranslator, BaseTranslator):
+class HTML5Translator(SphinxTranslator, BaseTranslator): # type: ignore[misc]
"""
Our custom HTML translator.
"""
+
builder: StandaloneHTMLBuilder
+ # Override docutils.writers.html5_polyglot:HTMLTranslator
+ # otherwise, nodes like <inline classes="s">...</inline> will be
+ # converted to <s>...</s> by `visit_inline`.
supported_inline_tags: set[str] = set()
- def __init__(self, document: nodes.document, builder: Builder) ->None:
+ def __init__(self, document: nodes.document, builder: Builder) -> None:
super().__init__(document, builder)
+
self.highlighter = self.builder.highlighter
- self.docnames = [self.builder.current_docname]
+ self.docnames = [self.builder.current_docname] # for singlehtml builder
self.protect_literal_text = 0
self.secnumber_suffix = self.config.html_secnumber_suffix
self.param_separator = ''
@@ -44,11 +67,881 @@ class HTML5Translator(SphinxTranslator, BaseTranslator):
self._fieldlist_row_indices = [0]
self.required_params_left = 0
- def _visit_sig_parameter_list(self, node: Element, parameter_group:
- type[Element], sig_open_paren: str, sig_close_paren: str) ->None:
+ def visit_start_of_file(self, node: Element) -> None:
+ # only occurs in the single-file builder
+ self.docnames.append(node['docname'])
+ self.body.append('<span id="document-%s"></span>' % node['docname'])
+
+ def depart_start_of_file(self, node: Element) -> None:
+ self.docnames.pop()
+
+ #############################################################
+ # Domain-specific object descriptions
+ #############################################################
+
+ # Top-level nodes for descriptions
+ ##################################
+
+ def visit_desc(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'dl'))
+
+ def depart_desc(self, node: Element) -> None:
+ self.body.append('</dl>\n\n')
+
+ def visit_desc_signature(self, node: Element) -> None:
+ # the id is set automatically
+ self.body.append(self.starttag(node, 'dt'))
+ self.protect_literal_text += 1
+
+ def depart_desc_signature(self, node: Element) -> None:
+ self.protect_literal_text -= 1
+ if not node.get('is_multiline'):
+ self.add_permalink_ref(node, _('Link to this definition'))
+ self.body.append('</dt>\n')
+
+ def visit_desc_signature_line(self, node: Element) -> None:
+ pass
+
+ def depart_desc_signature_line(self, node: Element) -> None:
+ if node.get('add_permalink'):
+ # the permalink info is on the parent desc_signature node
+ self.add_permalink_ref(node.parent, _('Link to this definition'))
+ self.body.append('<br />')
+
+ def visit_desc_content(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'dd', ''))
+
+ def depart_desc_content(self, node: Element) -> None:
+ self.body.append('</dd>')
+
+ def visit_desc_inline(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'span', ''))
+
+ def depart_desc_inline(self, node: Element) -> None:
+ self.body.append('</span>')
+
+ # Nodes for high-level structure in signatures
+ ##############################################
+
+ def visit_desc_name(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'span', ''))
+
+ def depart_desc_name(self, node: Element) -> None:
+ self.body.append('</span>')
+
+ def visit_desc_addname(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'span', ''))
+
+ def depart_desc_addname(self, node: Element) -> None:
+ self.body.append('</span>')
+
+ def visit_desc_type(self, node: Element) -> None:
+ pass
+
+ def depart_desc_type(self, node: Element) -> None:
+ pass
+
+ def visit_desc_returns(self, node: Element) -> None:
+ self.body.append(' <span class="sig-return">')
+ self.body.append('<span class="sig-return-icon">→</span>')
+ self.body.append(' <span class="sig-return-typehint">')
+
+ def depart_desc_returns(self, node: Element) -> None:
+ self.body.append('</span></span>')
+
+ def _visit_sig_parameter_list(
+ self,
+ node: Element,
+ parameter_group: type[Element],
+ sig_open_paren: str,
+ sig_close_paren: str,
+ ) -> None:
"""Visit a signature parameters or type parameters list.
The *parameter_group* value is the type of child nodes acting as required parameters
or as a set of contiguous optional parameters.
"""
+ self.body.append(f'<span class="sig-paren">{sig_open_paren}</span>')
+ self.is_first_param = True
+ self.optional_param_level = 0
+ self.params_left_at_level = 0
+ self.param_group_index = 0
+ # Counts as what we call a parameter group either a required parameter, or a
+ # set of contiguous optional ones.
+ self.list_is_required_param = [isinstance(c, parameter_group) for c in node.children]
+ # How many required parameters are left.
+ self.required_params_left = sum(self.list_is_required_param)
+ self.param_separator = node.child_text_separator
+ self.multi_line_parameter_list = node.get('multi_line_parameter_list', False)
+ if self.multi_line_parameter_list:
+ self.body.append('\n\n')
+ self.body.append(self.starttag(node, 'dl'))
+ self.param_separator = self.param_separator.rstrip()
+ self.context.append(sig_close_paren)
+
+ def _depart_sig_parameter_list(self, node: Element) -> None:
+ if node.get('multi_line_parameter_list'):
+ self.body.append('</dl>\n\n')
+ sig_close_paren = self.context.pop()
+ self.body.append(f'<span class="sig-paren">{sig_close_paren}</span>')
+
+ def visit_desc_parameterlist(self, node: Element) -> None:
+ self._visit_sig_parameter_list(node, addnodes.desc_parameter, '(', ')')
+
+ def depart_desc_parameterlist(self, node: Element) -> None:
+ self._depart_sig_parameter_list(node)
+
+ def visit_desc_type_parameter_list(self, node: Element) -> None:
+ self._visit_sig_parameter_list(node, addnodes.desc_type_parameter, '[', ']')
+
+ def depart_desc_type_parameter_list(self, node: Element) -> None:
+ self._depart_sig_parameter_list(node)
+
+ # If required parameters are still to come, then put the comma after
+ # the parameter. Otherwise, put the comma before. This ensures that
+ # signatures like the following render correctly (see issue #1001):
+ #
+ # foo([a, ]b, c[, d])
+ #
+ def visit_desc_parameter(self, node: Element) -> None:
+ on_separate_line = self.multi_line_parameter_list
+ if on_separate_line and not (self.is_first_param and self.optional_param_level > 0):
+ self.body.append(self.starttag(node, 'dd', ''))
+ if self.is_first_param:
+ self.is_first_param = False
+ elif not on_separate_line and not self.required_params_left:
+ self.body.append(self.param_separator)
+ if self.optional_param_level == 0:
+ self.required_params_left -= 1
+ else:
+ self.params_left_at_level -= 1
+ if not node.hasattr('noemph'):
+ self.body.append('<em class="sig-param">')
+
+ def depart_desc_parameter(self, node: Element) -> None:
+ if not node.hasattr('noemph'):
+ self.body.append('</em>')
+ is_required = self.list_is_required_param[self.param_group_index]
+ if self.multi_line_parameter_list:
+ is_last_group = self.param_group_index + 1 == len(self.list_is_required_param)
+ next_is_required = (
+ not is_last_group
+ and self.list_is_required_param[self.param_group_index + 1]
+ )
+ opt_param_left_at_level = self.params_left_at_level > 0
+ if opt_param_left_at_level or is_required and (is_last_group or next_is_required):
+ self.body.append(self.param_separator)
+ self.body.append('</dd>\n')
+
+ elif self.required_params_left:
+ self.body.append(self.param_separator)
+
+ if is_required:
+ self.param_group_index += 1
+
+ def visit_desc_type_parameter(self, node: Element) -> None:
+ self.visit_desc_parameter(node)
+
+ def depart_desc_type_parameter(self, node: Element) -> None:
+ self.depart_desc_parameter(node)
+
+ def visit_desc_optional(self, node: Element) -> None:
+ self.params_left_at_level = sum(isinstance(c, addnodes.desc_parameter)
+ for c in node.children)
+ self.optional_param_level += 1
+ self.max_optional_param_level = self.optional_param_level
+ if self.multi_line_parameter_list:
+ # If the first parameter is optional, start a new line and open the bracket.
+ if self.is_first_param:
+ self.body.append(self.starttag(node, 'dd', ''))
+ self.body.append('<span class="optional">[</span>')
+ # Else, if there remains at least one required parameter, append the
+ # parameter separator, open a new bracket, and end the line.
+ elif self.required_params_left:
+ self.body.append(self.param_separator)
+ self.body.append('<span class="optional">[</span>')
+ self.body.append('</dd>\n')
+ # Else, open a new bracket, append the parameter separator,
+ # and end the line.
+ else:
+ self.body.append('<span class="optional">[</span>')
+ self.body.append(self.param_separator)
+ self.body.append('</dd>\n')
+ else:
+ self.body.append('<span class="optional">[</span>')
+
+ def depart_desc_optional(self, node: Element) -> None:
+ self.optional_param_level -= 1
+ if self.multi_line_parameter_list:
+ # If it's the first time we go down one level, add the separator
+ # before the bracket.
+ if self.optional_param_level == self.max_optional_param_level - 1:
+ self.body.append(self.param_separator)
+ self.body.append('<span class="optional">]</span>')
+ # End the line if we have just closed the last bracket of this
+ # optional parameter group.
+ if self.optional_param_level == 0:
+ self.body.append('</dd>\n')
+ else:
+ self.body.append('<span class="optional">]</span>')
+ if self.optional_param_level == 0:
+ self.param_group_index += 1
+
+ def visit_desc_annotation(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'em', '', CLASS='property'))
+
+ def depart_desc_annotation(self, node: Element) -> None:
+ self.body.append('</em>')
+
+ ##############################################
+
+ def visit_versionmodified(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'div', CLASS=node['type']))
+
+ def depart_versionmodified(self, node: Element) -> None:
+ self.body.append('</div>\n')
+
+ # overwritten
+ def visit_reference(self, node: Element) -> None:
+ atts = {'class': 'reference'}
+ if node.get('internal') or 'refuri' not in node:
+ atts['class'] += ' internal'
+ else:
+ atts['class'] += ' external'
+ if 'refuri' in node:
+ atts['href'] = node['refuri'] or '#'
+ if self.settings.cloak_email_addresses and atts['href'].startswith('mailto:'):
+ atts['href'] = self.cloak_mailto(atts['href'])
+ self.in_mailto = True
+ else:
+ assert 'refid' in node, \
+ 'References must have "refuri" or "refid" attribute.'
+ atts['href'] = '#' + node['refid']
+ if not isinstance(node.parent, nodes.TextElement):
+ assert len(node) == 1 and isinstance(node[0], nodes.image) # NoQA: PT018
+ atts['class'] += ' image-reference'
+ if 'reftitle' in node:
+ atts['title'] = node['reftitle']
+ if 'target' in node:
+ atts['target'] = node['target']
+ if 'rel' in node:
+ atts['rel'] = node['rel']
+ self.body.append(self.starttag(node, 'a', '', **atts))
+
+ if node.get('secnumber'):
+ self.body.append(('%s' + self.secnumber_suffix) %
+ '.'.join(map(str, node['secnumber'])))
+
+ def visit_number_reference(self, node: Element) -> None:
+ self.visit_reference(node)
+
+ def depart_number_reference(self, node: Element) -> None:
+ self.depart_reference(node)
+
+ # overwritten -- we don't want source comments to show up in the HTML
+ def visit_comment(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ # overwritten
+ def visit_admonition(self, node: Element, name: str = '') -> None:
+ self.body.append(self.starttag(
+ node, 'div', CLASS=('admonition ' + name)))
+ if name:
+ node.insert(0, nodes.title(name, admonitionlabels[name]))
+
+ def depart_admonition(self, node: Element | None = None) -> None:
+ self.body.append('</div>\n')
+
+ def visit_seealso(self, node: Element) -> None:
+ self.visit_admonition(node, 'seealso')
+
+ def depart_seealso(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def get_secnumber(self, node: Element) -> tuple[int, ...] | None:
+ if node.get('secnumber'):
+ return node['secnumber']
+
+ if isinstance(node.parent, nodes.section):
+ if self.builder.name == 'singlehtml':
+ docname = self.docnames[-1]
+ anchorname = "{}/#{}".format(docname, node.parent['ids'][0])
+ if anchorname not in self.builder.secnumbers:
+ anchorname = "%s/" % docname # try first heading which has no anchor
+ else:
+ anchorname = '#' + node.parent['ids'][0]
+ if anchorname not in self.builder.secnumbers:
+ anchorname = '' # try first heading which has no anchor
+
+ if self.builder.secnumbers.get(anchorname):
+ return self.builder.secnumbers[anchorname]
+
+ return None
+
+ def add_secnumber(self, node: Element) -> None:
+ secnumber = self.get_secnumber(node)
+ if secnumber:
+ self.body.append('<span class="section-number">%s</span>' %
+ ('.'.join(map(str, secnumber)) + self.secnumber_suffix))
+
+ def add_fignumber(self, node: Element) -> None:
+ def append_fignumber(figtype: str, figure_id: str) -> None:
+ if self.builder.name == 'singlehtml':
+ key = f"{self.docnames[-1]}/{figtype}"
+ else:
+ key = figtype
+
+ if figure_id in self.builder.fignumbers.get(key, {}):
+ self.body.append('<span class="caption-number">')
+ prefix = self.config.numfig_format.get(figtype)
+ if prefix is None:
+ msg = __('numfig_format is not defined for %s') % figtype
+ logger.warning(msg)
+ else:
+ numbers = self.builder.fignumbers[key][figure_id]
+ self.body.append(prefix % '.'.join(map(str, numbers)) + ' ')
+ self.body.append('</span>')
+
+ figtype = self.builder.env.domains['std'].get_enumerable_node_type(node)
+ if figtype:
+ if len(node['ids']) == 0:
+ msg = __('Any IDs not assigned for %s node') % node.tagname
+ logger.warning(msg, location=node)
+ else:
+ append_fignumber(figtype, node['ids'][0])
+
+ def add_permalink_ref(self, node: Element, title: str) -> None:
+ icon = self.config.html_permalinks_icon
+ if node['ids'] and self.config.html_permalinks and self.builder.add_permalinks:
+ self.body.append(
+ f'<a class="headerlink" href="#{node["ids"][0]}" title="{title}">{icon}</a>',
+ )
+
+ # overwritten
+ def visit_bullet_list(self, node: Element) -> None:
+ if len(node) == 1 and isinstance(node[0], addnodes.toctree):
+ # avoid emitting empty <ul></ul>
+ raise nodes.SkipNode
+ super().visit_bullet_list(node)
+
+ # overwritten
+ def visit_definition(self, node: Element) -> None:
+ # don't insert </dt> here.
+ self.body.append(self.starttag(node, 'dd', ''))
+
+ # overwritten
+ def depart_definition(self, node: Element) -> None:
+ self.body.append('</dd>\n')
+
+ # overwritten
+ def visit_classifier(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'span', '', CLASS='classifier'))
+
+ # overwritten
+ def depart_classifier(self, node: Element) -> None:
+ self.body.append('</span>')
+
+ next_node: Node = node.next_node(descend=False, siblings=True)
+ if not isinstance(next_node, nodes.classifier):
+ # close `<dt>` tag at the tail of classifiers
+ self.body.append('</dt>')
+
+ # overwritten
+ def visit_term(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'dt', ''))
+
+ # overwritten
+ def depart_term(self, node: Element) -> None:
+ next_node: Node = node.next_node(descend=False, siblings=True)
+ if isinstance(next_node, nodes.classifier):
+ # Leave the end tag to `self.depart_classifier()`, in case
+ # there's a classifier.
+ pass
+ else:
+ if isinstance(node.parent.parent.parent, addnodes.glossary):
+ # add permalink if glossary terms
+ self.add_permalink_ref(node, _('Link to this term'))
+
+ self.body.append('</dt>')
+
+ # overwritten
+ def visit_title(self, node: Element) -> None:
+ if isinstance(node.parent, addnodes.compact_paragraph) and node.parent.get('toctree'):
+ self.body.append(self.starttag(node, 'p', '', CLASS='caption', ROLE='heading'))
+ self.body.append('<span class="caption-text">')
+ self.context.append('</span></p>\n')
+ else:
+ super().visit_title(node)
+ self.add_secnumber(node)
+ self.add_fignumber(node.parent)
+ if isinstance(node.parent, nodes.table):
+ self.body.append('<span class="caption-text">')
+ # Partially revert https://sourceforge.net/p/docutils/code/9562/
+ if (
+ isinstance(node.parent, nodes.topic)
+ and self.settings.toc_backlinks
+ and 'contents' in node.parent['classes']
+ and self.body[-1].startswith('<a ')
+ # TODO: only remove for EPUB
+ ):
+ # remove <a class="reference internal" href="#top">
+ self.body.pop()
+ self.context[-1] = '</p>\n'
+
+ def depart_title(self, node: Element) -> None:
+ close_tag = self.context[-1]
+ if (self.config.html_permalinks and self.builder.add_permalinks and
+ node.parent.hasattr('ids') and node.parent['ids']):
+ # add permalink anchor
+ if close_tag.startswith('</h'):
+ self.add_permalink_ref(node.parent, _('Link to this heading'))
+ elif close_tag.startswith('</a></h'):
+ self.body.append('</a><a class="headerlink" href="#%s" ' %
+ node.parent['ids'][0] +
+ 'title="{}">{}'.format(
+ _('Link to this heading'),
+ self.config.html_permalinks_icon))
+ elif isinstance(node.parent, nodes.table):
+ self.body.append('</span>')
+ self.add_permalink_ref(node.parent, _('Link to this table'))
+ elif isinstance(node.parent, nodes.table):
+ self.body.append('</span>')
+
+ super().depart_title(node)
+
+ # overwritten
+ def visit_rubric(self, node: nodes.rubric) -> None:
+ if 'heading-level' in node:
+ level = node['heading-level']
+ if level in {1, 2, 3, 4, 5, 6}:
+ self.body.append(self.starttag(node, f'h{level}', '', CLASS='rubric'))
+ else:
+ logger.warning(
+ __('unsupported rubric heading level: %s'),
+ level,
+ type='html',
+ location=node
+ )
+ super().visit_rubric(node)
+ else:
+ super().visit_rubric(node)
+
+ # overwritten
+ def depart_rubric(self, node: nodes.rubric) -> None:
+ if (level := node.get('heading-level')) in {1, 2, 3, 4, 5, 6}:
+ self.body.append(f'</h{level}>\n')
+ else:
+ super().depart_rubric(node)
+
+ # overwritten
+ def visit_literal_block(self, node: Element) -> None:
+ if node.rawsource != node.astext():
+ # most probably a parsed-literal block -- don't highlight
+ return super().visit_literal_block(node)
+
+ lang = node.get('language', 'default')
+ linenos = node.get('linenos', False)
+ highlight_args = node.get('highlight_args', {})
+ highlight_args['force'] = node.get('force', False)
+ opts = self.config.highlight_options.get(lang, {})
+
+ if linenos and self.config.html_codeblock_linenos_style:
+ linenos = self.config.html_codeblock_linenos_style
+
+ highlighted = self.highlighter.highlight_block(
+ node.rawsource, lang, opts=opts, linenos=linenos,
+ location=node, **highlight_args,
+ )
+ starttag = self.starttag(node, 'div', suffix='',
+ CLASS='highlight-%s notranslate' % lang)
+ self.body.append(starttag + highlighted + '</div>\n')
+ raise nodes.SkipNode
+
+ def visit_caption(self, node: Element) -> None:
+ if isinstance(node.parent, nodes.container) and node.parent.get('literal_block'):
+ self.body.append('<div class="code-block-caption">')
+ else:
+ super().visit_caption(node)
+ self.add_fignumber(node.parent)
+ self.body.append(self.starttag(node, 'span', '', CLASS='caption-text'))
+
+ def depart_caption(self, node: Element) -> None:
+ self.body.append('</span>')
+
+ # append permalink if available
+ if isinstance(node.parent, nodes.container) and node.parent.get('literal_block'):
+ self.add_permalink_ref(node.parent, _('Link to this code'))
+ elif isinstance(node.parent, nodes.figure):
+ self.add_permalink_ref(node.parent, _('Link to this image'))
+ elif node.parent.get('toctree'):
+ self.add_permalink_ref(node.parent.parent, _('Link to this toctree'))
+
+ if isinstance(node.parent, nodes.container) and node.parent.get('literal_block'):
+ self.body.append('</div>\n')
+ else:
+ super().depart_caption(node)
+
+ def visit_doctest_block(self, node: Element) -> None:
+ self.visit_literal_block(node)
+
+ # overwritten to add the <div> (for XHTML compliance)
+ def visit_block_quote(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'blockquote') + '<div>')
+
+ def depart_block_quote(self, node: Element) -> None:
+ self.body.append('</div></blockquote>\n')
+
+ # overwritten
+ def visit_literal(self, node: Element) -> None:
+ if 'kbd' in node['classes']:
+ self.body.append(self.starttag(node, 'kbd', '',
+ CLASS='docutils literal notranslate'))
+ return
+ lang = node.get("language", None)
+ if 'code' not in node['classes'] or not lang:
+ self.body.append(self.starttag(node, 'code', '',
+ CLASS='docutils literal notranslate'))
+ self.protect_literal_text += 1
+ return
+
+ opts = self.config.highlight_options.get(lang, {})
+ highlighted = self.highlighter.highlight_block(
+ node.astext(), lang, opts=opts, location=node, nowrap=True)
+ starttag = self.starttag(
+ node,
+ "code",
+ suffix="",
+ CLASS="docutils literal highlight highlight-%s" % lang,
+ )
+ self.body.append(starttag + highlighted.strip() + "</code>")
+ raise nodes.SkipNode
+
+ def depart_literal(self, node: Element) -> None:
+ if 'kbd' in node['classes']:
+ self.body.append('</kbd>')
+ else:
+ self.protect_literal_text -= 1
+ self.body.append('</code>')
+
+ def visit_productionlist(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'pre'))
+ productionlist = cast(Iterable[addnodes.production], node)
+ names = (production['tokenname'] for production in productionlist)
+ maxlen = max(len(name) for name in names)
+ lastname = None
+ for production in productionlist:
+ if production['tokenname']:
+ lastname = production['tokenname'].ljust(maxlen)
+ self.body.append(self.starttag(production, 'strong', ''))
+ self.body.append(lastname + '</strong> ::= ')
+ elif lastname is not None:
+ self.body.append('%s ' % (' ' * len(lastname)))
+ production.walkabout(self)
+ self.body.append('\n')
+ self.body.append('</pre>\n')
+ raise nodes.SkipNode
+
+ def depart_productionlist(self, node: Element) -> None:
+ pass
+
+ def visit_production(self, node: Element) -> None:
+ pass
+
+ def depart_production(self, node: Element) -> None:
pass
+
+ def visit_centered(self, node: Element) -> None:
+ self.body.append(self.starttag(node, 'p', CLASS="centered") +
+ '<strong>')
+
+ def depart_centered(self, node: Element) -> None:
+ self.body.append('</strong></p>')
+
+ def visit_compact_paragraph(self, node: Element) -> None:
+ pass
+
+ def depart_compact_paragraph(self, node: Element) -> None:
+ pass
+
+ def visit_download_reference(self, node: Element) -> None:
+ atts = {'class': 'reference download',
+ 'download': ''}
+
+ if not self.builder.download_support:
+ self.context.append('')
+ elif 'refuri' in node:
+ atts['class'] += ' external'
+ atts['href'] = node['refuri']
+ self.body.append(self.starttag(node, 'a', '', **atts))
+ self.context.append('</a>')
+ elif 'filename' in node:
+ atts['class'] += ' internal'
+ atts['href'] = posixpath.join(self.builder.dlpath,
+ urllib.parse.quote(node['filename']))
+ self.body.append(self.starttag(node, 'a', '', **atts))
+ self.context.append('</a>')
+ else:
+ self.context.append('')
+
+ def depart_download_reference(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+
+ # overwritten
+ def visit_figure(self, node: Element) -> None:
+ # set align=default if align not specified to give a default style
+ node.setdefault('align', 'default')
+
+ return super().visit_figure(node)
+
+ # overwritten
+ def visit_image(self, node: Element) -> None:
+ olduri = node['uri']
+ # rewrite the URI if the environment knows about it
+ if olduri in self.builder.images:
+ node['uri'] = posixpath.join(self.builder.imgpath,
+ urllib.parse.quote(self.builder.images[olduri]))
+
+ if 'scale' in node:
+ # Try to figure out image height and width. Docutils does that too,
+ # but it tries the final file name, which does not necessarily exist
+ # yet at the time the HTML file is written.
+ if not ('width' in node and 'height' in node):
+ path = os.path.join(self.builder.srcdir, olduri) # type: ignore[has-type]
+ size = get_image_size(path)
+ if size is None:
+ logger.warning(
+ __('Could not obtain image size. :scale: option is ignored.'),
+ location=node,
+ )
+ else:
+ if 'width' not in node:
+ node['width'] = str(size[0])
+ if 'height' not in node:
+ node['height'] = str(size[1])
+
+ super().visit_image(node)
+
+ # overwritten
+ def depart_image(self, node: Element) -> None:
+ if node['uri'].lower().endswith(('svg', 'svgz')):
+ pass
+ else:
+ super().depart_image(node)
+
+ def visit_toctree(self, node: Element) -> None:
+ # this only happens when formatting a toc from env.tocs -- in this
+ # case we don't want to include the subtree
+ raise nodes.SkipNode
+
+ def visit_index(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_tabular_col_spec(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_glossary(self, node: Element) -> None:
+ pass
+
+ def depart_glossary(self, node: Element) -> None:
+ pass
+
+ def visit_acks(self, node: Element) -> None:
+ pass
+
+ def depart_acks(self, node: Element) -> None:
+ pass
+
+ def visit_hlist(self, node: Element) -> None:
+ self.body.append('<table class="hlist"><tr>')
+
+ def depart_hlist(self, node: Element) -> None:
+ self.body.append('</tr></table>\n')
+
+ def visit_hlistcol(self, node: Element) -> None:
+ self.body.append('<td>')
+
+ def depart_hlistcol(self, node: Element) -> None:
+ self.body.append('</td>')
+
+ # overwritten
+ def visit_Text(self, node: Text) -> None:
+ text = node.astext()
+ encoded = self.encode(text)
+ if self.protect_literal_text:
+ # moved here from base class's visit_literal to support
+ # more formatting in literal nodes
+ for token in self.words_and_spaces.findall(encoded):
+ if token.strip():
+ # protect literal text from line wrapping
+ self.body.append('<span class="pre">%s</span>' % token)
+ elif token in ' \n':
+ # allow breaks at whitespace
+ self.body.append(token)
+ else:
+ # protect runs of multiple spaces; the last one can wrap
+ self.body.append(' ' * (len(token) - 1) + ' ')
+ else:
+ if self.in_mailto and self.settings.cloak_email_addresses:
+ encoded = self.cloak_email(encoded)
+ self.body.append(encoded)
+
+ def visit_note(self, node: Element) -> None:
+ self.visit_admonition(node, 'note')
+
+ def depart_note(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_warning(self, node: Element) -> None:
+ self.visit_admonition(node, 'warning')
+
+ def depart_warning(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_attention(self, node: Element) -> None:
+ self.visit_admonition(node, 'attention')
+
+ def depart_attention(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_caution(self, node: Element) -> None:
+ self.visit_admonition(node, 'caution')
+
+ def depart_caution(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_danger(self, node: Element) -> None:
+ self.visit_admonition(node, 'danger')
+
+ def depart_danger(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_error(self, node: Element) -> None:
+ self.visit_admonition(node, 'error')
+
+ def depart_error(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_hint(self, node: Element) -> None:
+ self.visit_admonition(node, 'hint')
+
+ def depart_hint(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_important(self, node: Element) -> None:
+ self.visit_admonition(node, 'important')
+
+ def depart_important(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_tip(self, node: Element) -> None:
+ self.visit_admonition(node, 'tip')
+
+ def depart_tip(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_literal_emphasis(self, node: Element) -> None:
+ return self.visit_emphasis(node)
+
+ def depart_literal_emphasis(self, node: Element) -> None:
+ return self.depart_emphasis(node)
+
+ def visit_literal_strong(self, node: Element) -> None:
+ return self.visit_strong(node)
+
+ def depart_literal_strong(self, node: Element) -> None:
+ return self.depart_strong(node)
+
+ def visit_abbreviation(self, node: Element) -> None:
+ attrs = {}
+ if node.hasattr('explanation'):
+ attrs['title'] = node['explanation']
+ self.body.append(self.starttag(node, 'abbr', '', **attrs))
+
+ def depart_abbreviation(self, node: Element) -> None:
+ self.body.append('</abbr>')
+
+ def visit_manpage(self, node: Element) -> None:
+ self.visit_literal_emphasis(node)
+
+ def depart_manpage(self, node: Element) -> None:
+ self.depart_literal_emphasis(node)
+
+ # overwritten to add even/odd classes
+
+ def visit_table(self, node: Element) -> None:
+ self._table_row_indices.append(0)
+
+ atts = {}
+ classes = [cls.strip(' \t\n') for cls in self.settings.table_style.split(',')]
+ classes.insert(0, "docutils") # compat
+
+ # set align-default if align not specified to give a default style
+ classes.append('align-%s' % node.get('align', 'default'))
+
+ if 'width' in node:
+ atts['style'] = 'width: %s' % node['width']
+ tag = self.starttag(node, 'table', CLASS=' '.join(classes), **atts)
+ self.body.append(tag)
+
+ def depart_table(self, node: Element) -> None:
+ self._table_row_indices.pop()
+ super().depart_table(node)
+
+ def visit_row(self, node: Element) -> None:
+ self._table_row_indices[-1] += 1
+ if self._table_row_indices[-1] % 2 == 0:
+ node['classes'].append('row-even')
+ else:
+ node['classes'].append('row-odd')
+ self.body.append(self.starttag(node, 'tr', ''))
+ node.column = 0 # type: ignore[attr-defined]
+
+ def visit_field_list(self, node: Element) -> None:
+ self._fieldlist_row_indices.append(0)
+ return super().visit_field_list(node)
+
+ def depart_field_list(self, node: Element) -> None:
+ self._fieldlist_row_indices.pop()
+ return super().depart_field_list(node)
+
+ def visit_field(self, node: Element) -> None:
+ self._fieldlist_row_indices[-1] += 1
+ if self._fieldlist_row_indices[-1] % 2 == 0:
+ node['classes'].append('field-even')
+ else:
+ node['classes'].append('field-odd')
+
+ def visit_math(self, node: Element, math_env: str = '') -> None:
+ # see validate_math_renderer
+ name: str = self.builder.math_renderer_name # type: ignore[assignment]
+ visit, _ = self.builder.app.registry.html_inline_math_renderers[name]
+ visit(self, node)
+
+ def depart_math(self, node: Element, math_env: str = '') -> None:
+ # see validate_math_renderer
+ name: str = self.builder.math_renderer_name # type: ignore[assignment]
+ _, depart = self.builder.app.registry.html_inline_math_renderers[name]
+ if depart:
+ depart(self, node)
+
+ def visit_math_block(self, node: Element, math_env: str = '') -> None:
+ # see validate_math_renderer
+ name: str = self.builder.math_renderer_name # type: ignore[assignment]
+ visit, _ = self.builder.app.registry.html_block_math_renderers[name]
+ visit(self, node)
+
+ def depart_math_block(self, node: Element, math_env: str = '') -> None:
+ # see validate_math_renderer
+ name: str = self.builder.math_renderer_name # type: ignore[assignment]
+ _, depart = self.builder.app.registry.html_block_math_renderers[name]
+ if depart:
+ depart(self, node)
+
+ # See Docutils r9413
+ # Re-instate the footnote-reference class
+ def visit_footnote_reference(self, node: Element) -> None:
+ href = '#' + node['refid']
+ classes = ['footnote-reference', self.settings.footnote_references]
+ self.body.append(self.starttag(node, 'a', suffix='', classes=classes,
+ role='doc-noteref', href=href))
+ self.body.append('<span class="fn-bracket">[</span>')
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index c0c3fff2d..0f2ffc29f 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -3,13 +3,17 @@
Much of this code is adapted from Dave Kuhlman's "docpy" writer from his
docutils sandbox.
"""
+
from __future__ import annotations
+
import re
from collections import defaultdict
from collections.abc import Iterable
from os import path
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes, writers
+
from sphinx import addnodes, highlighting
from sphinx.domains.std import StandardDomain
from sphinx.errors import SphinxError
@@ -20,25 +24,38 @@ from sphinx.util.index_entries import split_index_msg
from sphinx.util.nodes import clean_astext, get_prev_node
from sphinx.util.template import LaTeXRenderer
from sphinx.util.texescape import tex_replace_map
+
try:
from docutils.utils.roman import toRoman
except ImportError:
- from roman import toRoman
+ # In Debian/Ubuntu, roman package is provided as roman, not as docutils.utils.roman
+ from roman import toRoman # type: ignore[no-redef, import-not-found]
+
if TYPE_CHECKING:
from docutils.nodes import Element, Node, Text
+
from sphinx.builders.latex import LaTeXBuilder
from sphinx.builders.latex.theming import Theme
from sphinx.domains import IndexEntry
+
+
logger = logging.getLogger(__name__)
+
MAX_CITATION_LABEL_LENGTH = 8
-LATEXSECTIONNAMES = ['part', 'chapter', 'section', 'subsection',
- 'subsubsection', 'paragraph', 'subparagraph']
-ENUMERATE_LIST_STYLE = defaultdict(lambda : '\\arabic', {'arabic':
- '\\arabic', 'loweralpha': '\\alph', 'upperalpha': '\\Alph',
- 'lowerroman': '\\roman', 'upperroman': '\\Roman'})
+LATEXSECTIONNAMES = ["part", "chapter", "section", "subsection",
+ "subsubsection", "paragraph", "subparagraph"]
+ENUMERATE_LIST_STYLE = defaultdict(lambda: r'\arabic',
+ {
+ 'arabic': r'\arabic',
+ 'loweralpha': r'\alph',
+ 'upperalpha': r'\Alph',
+ 'lowerroman': r'\roman',
+ 'upperroman': r'\Roman',
+ })
+
CR = '\n'
BLANKLINE = '\n\n'
-EXTRA_RE = re.compile('^(.*\\S)\\s+\\(([^()]*)\\)\\s*$')
+EXTRA_RE = re.compile(r'^(.*\S)\s+\(([^()]*)\)\s*$')
class collected_footnote(nodes.footnote):
@@ -49,23 +66,35 @@ class UnsupportedError(SphinxError):
category = 'Markup is unsupported in LaTeX'
-class LaTeXWriter(writers.Writer):
- supported = 'sphinxlatex',
- settings_spec = 'LaTeX writer options', '', (('Document name', [
- '--docname'], {'default': ''}), ('Document class', ['--docclass'],
- {'default': 'manual'}), ('Author', ['--author'], {'default': ''}))
+class LaTeXWriter(writers.Writer): # type: ignore[misc]
+
+ supported = ('sphinxlatex',)
+
+ settings_spec = ('LaTeX writer options', '', (
+ ('Document name', ['--docname'], {'default': ''}),
+ ('Document class', ['--docclass'], {'default': 'manual'}),
+ ('Author', ['--author'], {'default': ''}),
+ ))
settings_defaults: dict[str, Any] = {}
+
theme: Theme
- def __init__(self, builder: LaTeXBuilder) ->None:
+ def __init__(self, builder: LaTeXBuilder) -> None:
super().__init__()
self.builder = builder
+ def translate(self) -> None:
+ visitor = self.builder.create_translator(self.document, self.builder, self.theme)
+ self.document.walkabout(visitor)
+ self.output = cast(LaTeXTranslator, visitor).astext()
+
+
+# Helper classes
class Table:
"""A table data"""
- def __init__(self, node: Element) ->None:
+ def __init__(self, node: Element) -> None:
self.header: list[str] = []
self.body: list[str] = []
self.align = node.get('align', 'default')
@@ -95,16 +124,20 @@ class Table:
self.has_verbatim = False
self.caption: list[str] = []
self.stubs: list[int] = []
+
+ # current position
self.col = 0
self.row = 0
+
+ # A dict mapping a table location to a cell_id (cell = rectangular area)
self.cells: dict[tuple[int, int], int] = defaultdict(int)
- self.cell_id = 0
+ self.cell_id = 0 # last assigned cell_id
- def is_longtable(self) ->bool:
+ def is_longtable(self) -> bool:
"""True if and only if table uses longtable environment."""
- pass
+ return self.row > 30 or 'longtable' in self.classes
- def get_table_type(self) ->str:
+ def get_table_type(self) -> str:
"""Returns the LaTeX environment name for the table.
The class currently supports:
@@ -113,83 +146,159 @@ class Table:
* tabular
* tabulary
"""
- pass
+ if self.is_longtable():
+ return 'longtable'
+ elif self.has_verbatim:
+ return 'tabular'
+ elif self.colspec:
+ return 'tabulary'
+ elif self.has_problematic or (self.colwidths and 'colwidths-given' in self.classes):
+ return 'tabular'
+ else:
+ return 'tabulary'
- def get_colspec(self) ->str:
- """Returns a column spec of table.
+ def get_colspec(self) -> str:
+ r"""Returns a column spec of table.
This is what LaTeX calls the 'preamble argument' of the used table environment.
.. note::
- The ``\\\\X`` and ``T`` column type specifiers are defined in
+ The ``\\X`` and ``T`` column type specifiers are defined in
``sphinxlatextables.sty``.
"""
- pass
+ if self.colspec:
+ return self.colspec
+
+ _colsep = self.colsep
+ assert _colsep is not None
+ if self.colwidths and 'colwidths-given' in self.classes:
+ total = sum(self.colwidths)
+ colspecs = [r'\X{%d}{%d}' % (width, total) for width in self.colwidths]
+ return f'{{{_colsep}{_colsep.join(colspecs)}{_colsep}}}' + CR
+ elif self.has_problematic:
+ return r'{%s*{%d}{\X{1}{%d}%s}}' % (_colsep, self.colcount,
+ self.colcount, _colsep) + CR
+ elif self.get_table_type() == 'tabulary':
+ # sphinx.sty sets T to be J by default.
+ return '{' + _colsep + (('T' + _colsep) * self.colcount) + '}' + CR
+ elif self.has_oldproblematic:
+ return r'{%s*{%d}{\X{1}{%d}%s}}' % (_colsep, self.colcount,
+ self.colcount, _colsep) + CR
+ else:
+ return '{' + _colsep + (('l' + _colsep) * self.colcount) + '}' + CR
- def add_cell(self, height: int, width: int) ->None:
+ def add_cell(self, height: int, width: int) -> None:
"""Adds a new cell to a table.
It will be located at current position: (``self.row``, ``self.col``).
"""
- pass
+ self.cell_id += 1
+ for col in range(width):
+ for row in range(height):
+ assert self.cells[(self.row + row, self.col + col)] == 0
+ self.cells[(self.row + row, self.col + col)] = self.cell_id
- def cell(self, row: (int | None)=None, col: (int | None)=None) ->(TableCell
- | None):
+ def cell(
+ self, row: int | None = None, col: int | None = None,
+ ) -> TableCell | None:
"""Returns a cell object (i.e. rectangular area) containing given position.
If no option arguments: ``row`` or ``col`` are given, the current position;
``self.row`` and ``self.col`` are used to get a cell object by default.
"""
- pass
+ try:
+ if row is None:
+ row = self.row
+ if col is None:
+ col = self.col
+ return TableCell(self, row, col)
+ except IndexError:
+ return None
class TableCell:
"""Data of a cell in a table."""
- def __init__(self, table: Table, row: int, col: int) ->None:
- if table.cells[row, col] == 0:
+ def __init__(self, table: Table, row: int, col: int) -> None:
+ if table.cells[(row, col)] == 0:
raise IndexError
+
self.table = table
- self.cell_id = table.cells[row, col]
+ self.cell_id = table.cells[(row, col)]
self.row = row
self.col = col
- while table.cells[self.row - 1, self.col] == self.cell_id:
+
+ # adjust position for multirow/multicol cell
+ while table.cells[(self.row - 1, self.col)] == self.cell_id:
self.row -= 1
- while table.cells[self.row, self.col - 1] == self.cell_id:
+ while table.cells[(self.row, self.col - 1)] == self.cell_id:
self.col -= 1
@property
- def width(self) ->int:
+ def width(self) -> int:
"""Returns the cell width."""
- pass
+ width = 0
+ while self.table.cells[(self.row, self.col + width)] == self.cell_id:
+ width += 1
+ return width
@property
- def height(self) ->int:
+ def height(self) -> int:
"""Returns the cell height."""
- pass
+ height = 0
+ while self.table.cells[(self.row + height, self.col)] == self.cell_id:
+ height += 1
+ return height
-def escape_abbr(text: str) ->str:
+def escape_abbr(text: str) -> str:
"""Adjust spacing after abbreviations."""
- pass
+ return re.sub(r'\.(?=\s|$)', r'.\@', text)
-def rstdim_to_latexdim(width_str: str, scale: int=100) ->str:
+def rstdim_to_latexdim(width_str: str, scale: int = 100) -> str:
"""Convert `width_str` with rst length to LaTeX length."""
- pass
+ match = re.match(r'^(\d*\.?\d*)\s*(\S*)$', width_str)
+ if not match:
+ raise ValueError
+ res = width_str
+ amount, unit = match.groups()[:2]
+ if scale == 100:
+ float(amount) # validate amount is float
+ if unit in ('', "px"):
+ res = r"%s\sphinxpxdimen" % amount
+ elif unit == 'pt':
+ res = '%sbp' % amount # convert to 'bp'
+ elif unit == "%":
+ res = r"%.3f\linewidth" % (float(amount) / 100.0)
+ else:
+ amount_float = float(amount) * scale / 100.0
+ if unit in ('', "px"):
+ res = r"%.5f\sphinxpxdimen" % amount_float
+ elif unit == 'pt':
+ res = '%.5fbp' % amount_float
+ elif unit == "%":
+ res = r"%.5f\linewidth" % (amount_float / 100.0)
+ else:
+ res = f"{amount_float:.5f}{unit}"
+ return res
class LaTeXTranslator(SphinxTranslator):
builder: LaTeXBuilder
- secnumdepth = 2
+
+ secnumdepth = 2 # legacy sphinxhowto.cls uses this, whereas article.cls
+ # default is originally 3. For book/report, 2 is already LaTeX default.
ignore_missing_images = False
def __init__(self, document: nodes.document, builder: LaTeXBuilder,
- theme: Theme) ->None:
+ theme: Theme) -> None:
super().__init__(document, builder)
self.body: list[str] = []
self.theme = theme
+
+ # flags
self.in_title = 0
self.in_production_list = 0
self.in_footnote = 0
@@ -197,6 +306,7 @@ class LaTeXTranslator(SphinxTranslator):
self.in_term = 0
self.needs_linetrimming = 0
self.in_minipage = 0
+ # only used by figure inside an admonition
self.no_latex_floats = 0
self.first_document = 1
self.this_is_the_title = 1
@@ -205,81 +315,108 @@ class LaTeXTranslator(SphinxTranslator):
self.compact_list = 0
self.first_param = 0
self.in_desc_signature = False
+
sphinxpkgoptions = []
+
+ # sort out some elements
self.elements = self.builder.context.copy()
+
+ # initial section names
self.sectionnames = LATEXSECTIONNAMES.copy()
if self.theme.toplevel_sectioning == 'section':
self.sectionnames.remove('chapter')
+
+ # determine top section level
self.top_sectionlevel = 1
if self.config.latex_toplevel_sectioning:
try:
- self.top_sectionlevel = self.sectionnames.index(self.config
- .latex_toplevel_sectioning)
+ self.top_sectionlevel = \
+ self.sectionnames.index(self.config.latex_toplevel_sectioning)
except ValueError:
- logger.warning(__(
- 'unknown %r toplevel_sectioning for class %r'), self.
- config.latex_toplevel_sectioning, self.theme.docclass)
+ logger.warning(__('unknown %r toplevel_sectioning for class %r'),
+ self.config.latex_toplevel_sectioning, self.theme.docclass)
+
if self.config.numfig:
self.numfig_secnum_depth = self.config.numfig_secnum_depth
- if self.numfig_secnum_depth > 0:
- if len(self.sectionnames) < len(LATEXSECTIONNAMES
- ) and self.top_sectionlevel > 0:
+ if self.numfig_secnum_depth > 0: # default is 1
+ # numfig_secnum_depth as passed to sphinx.sty indices same names as in
+ # LATEXSECTIONNAMES but with -1 for part, 0 for chapter, 1 for section...
+ if len(self.sectionnames) < len(LATEXSECTIONNAMES) and \
+ self.top_sectionlevel > 0:
self.numfig_secnum_depth += self.top_sectionlevel
else:
self.numfig_secnum_depth += self.top_sectionlevel - 1
- self.numfig_secnum_depth = min(self.numfig_secnum_depth,
- len(LATEXSECTIONNAMES) - 1)
- sphinxpkgoptions.append('numfigreset=%s' % self.
- numfig_secnum_depth)
+ # this (minus one) will serve as minimum to LaTeX's secnumdepth
+ self.numfig_secnum_depth = min(self.numfig_secnum_depth,
+ len(LATEXSECTIONNAMES) - 1)
+ # if passed key value is < 1 LaTeX will act as if 0; see sphinx.sty
+ sphinxpkgoptions.append('numfigreset=%s' % self.numfig_secnum_depth)
else:
sphinxpkgoptions.append('nonumfigreset')
+
if self.config.numfig and self.config.math_numfig:
- sphinxpkgoptions.extend(['mathnumfig', 'mathnumsep={%s}' % self
- .config.math_numsep])
- if self.config.language not in {'en', 'ja'
- } and 'fncychap' not in self.config.latex_elements:
- self.elements['fncychap'] = ('\\usepackage[Sonny]{fncychap}' +
- CR + '\\ChNameVar{\\Large\\normalfont\\sffamily}' + CR +
- '\\ChTitleVar{\\Large\\normalfont\\sffamily}')
+ sphinxpkgoptions.extend([
+ 'mathnumfig',
+ 'mathnumsep={%s}' % self.config.math_numsep,
+ ])
+
+ if (self.config.language not in {'en', 'ja'} and
+ 'fncychap' not in self.config.latex_elements):
+ # use Sonny style if any language specified (except English)
+ self.elements['fncychap'] = (r'\usepackage[Sonny]{fncychap}' + CR +
+ r'\ChNameVar{\Large\normalfont\sffamily}' + CR +
+ r'\ChTitleVar{\Large\normalfont\sffamily}')
+
self.babel = self.builder.babel
if not self.babel.is_supported_language():
+ # emit warning if specified language is invalid
+ # (only emitting, nothing changed to processing)
logger.warning(__('no Babel option known for language %r'),
- self.config.language)
- minsecnumdepth = self.secnumdepth
+ self.config.language)
+
+ minsecnumdepth = self.secnumdepth # 2 from legacy sphinx manual/howto
if self.document.get('tocdepth'):
- tocdepth = self.document.get('tocdepth', 999
- ) + self.top_sectionlevel - 2
- if len(self.sectionnames) < len(LATEXSECTIONNAMES
- ) and self.top_sectionlevel > 0:
- tocdepth += 1
- if tocdepth > len(LATEXSECTIONNAMES) - 2:
+ # reduce tocdepth if `part` or `chapter` is used for top_sectionlevel
+ # tocdepth = -1: show only parts
+ # tocdepth = 0: show parts and chapters
+ # tocdepth = 1: show parts, chapters and sections
+ # tocdepth = 2: show parts, chapters, sections and subsections
+ # ...
+ tocdepth = self.document.get('tocdepth', 999) + self.top_sectionlevel - 2
+ if len(self.sectionnames) < len(LATEXSECTIONNAMES) and \
+ self.top_sectionlevel > 0:
+ tocdepth += 1 # because top_sectionlevel is shifted by -1
+ if tocdepth > len(LATEXSECTIONNAMES) - 2: # default is 5 <-> subparagraph
logger.warning(__('too large :maxdepth:, ignored.'))
tocdepth = len(LATEXSECTIONNAMES) - 2
- self.elements['tocdepth'] = '\\setcounter{tocdepth}{%d}' % tocdepth
+
+ self.elements['tocdepth'] = r'\setcounter{tocdepth}{%d}' % tocdepth
minsecnumdepth = max(minsecnumdepth, tocdepth)
- if self.config.numfig and self.config.numfig_secnum_depth > 0:
+
+ if self.config.numfig and (self.config.numfig_secnum_depth > 0):
minsecnumdepth = max(minsecnumdepth, self.numfig_secnum_depth - 1)
+
if minsecnumdepth > self.secnumdepth:
- self.elements['secnumdepth'
- ] = '\\setcounter{secnumdepth}{%d}' % minsecnumdepth
+ self.elements['secnumdepth'] = r'\setcounter{secnumdepth}{%d}' %\
+ minsecnumdepth
+
contentsname = document.get('contentsname')
if contentsname:
- self.elements['contentsname'] = self.babel_renewcommand(
- '\\contentsname', contentsname)
+ self.elements['contentsname'] = self.babel_renewcommand(r'\contentsname',
+ contentsname)
+
if self.elements['maxlistdepth']:
- sphinxpkgoptions.append('maxlistdepth=%s' % self.elements[
- 'maxlistdepth'])
+ sphinxpkgoptions.append('maxlistdepth=%s' % self.elements['maxlistdepth'])
if sphinxpkgoptions:
- self.elements['sphinxpkgoptions'] = '[,%s]' % ','.join(
- sphinxpkgoptions)
+ self.elements['sphinxpkgoptions'] = '[,%s]' % ','.join(sphinxpkgoptions)
if self.elements['sphinxsetup']:
- self.elements['sphinxsetup'] = '\\sphinxsetup{%s}' % self.elements[
- 'sphinxsetup']
+ self.elements['sphinxsetup'] = (r'\sphinxsetup{%s}' % self.elements['sphinxsetup'])
if self.elements['extraclassoptions']:
- self.elements['classoptions'] += ',' + self.elements[
- 'extraclassoptions']
- self.highlighter = highlighting.PygmentsBridge('latex', self.config
- .pygments_style, latex_engine=self.config.latex_engine)
+ self.elements['classoptions'] += ',' + \
+ self.elements['extraclassoptions']
+
+ self.highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style,
+ latex_engine=self.config.latex_engine)
self.context: list[Any] = []
self.descstack: list[str] = []
self.tables: list[Table] = []
@@ -290,14 +427,433 @@ class LaTeXTranslator(SphinxTranslator):
self.curfilestack: list[str] = []
self.handled_abbrs: set[str] = set()
+ def pushbody(self, newbody: list[str]) -> None:
+ self.bodystack.append(self.body)
+ self.body = newbody
+
+ def popbody(self) -> list[str]:
+ body = self.body
+ self.body = self.bodystack.pop()
+ return body
+
+ def astext(self) -> str:
+ self.elements.update({
+ 'body': ''.join(self.body),
+ 'indices': self.generate_indices(),
+ })
+ return self.render('latex.tex.jinja', self.elements)
+
+ def hypertarget(self, id: str, withdoc: bool = True, anchor: bool = True) -> str:
+ if withdoc:
+ id = self.curfilestack[-1] + ':' + id
+ return (r'\phantomsection' if anchor else '') + r'\label{%s}' % self.idescape(id)
+
+ def hypertarget_to(self, node: Element, anchor: bool = False) -> str:
+ labels = ''.join(self.hypertarget(node_id, anchor=False) for node_id in node['ids'])
+ if anchor:
+ return r'\phantomsection' + labels
+ else:
+ return labels
+
+ def hyperlink(self, id: str) -> str:
+ return r'{\hyperref[%s]{' % self.idescape(id)
+
+ def hyperpageref(self, id: str) -> str:
+ return r'\autopageref*{%s}' % self.idescape(id)
+
+ def escape(self, s: str) -> str:
+ return texescape.escape(s, self.config.latex_engine)
+
+ def idescape(self, id: str) -> str:
+ return r'\detokenize{%s}' % str(id).translate(tex_replace_map).\
+ encode('ascii', 'backslashreplace').decode('ascii').\
+ replace('\\', '_')
+
+ def babel_renewcommand(self, command: str, definition: str) -> str:
+ if self.elements['multilingual']:
+ prefix = r'\addto\captions%s{' % self.babel.get_language()
+ suffix = '}'
+ else: # babel is disabled (mainly for Japanese environment)
+ prefix = ''
+ suffix = ''
+
+ return fr'{prefix}\renewcommand{{{command}}}{{{definition}}}{suffix}' + CR
+
+ def generate_indices(self) -> str:
+ def generate(content: list[tuple[str, list[IndexEntry]]], collapsed: bool) -> None:
+ ret.append(r'\begin{sphinxtheindex}' + CR)
+ ret.append(r'\let\bigletter\sphinxstyleindexlettergroup' + CR)
+ for i, (letter, entries) in enumerate(content):
+ if i > 0:
+ ret.append(r'\indexspace' + CR)
+ ret.append(r'\bigletter{%s}' % self.escape(letter) + CR)
+ for entry in entries:
+ if not entry[3]:
+ continue
+ ret.append(r'\item\relax\sphinxstyleindexentry{%s}' %
+ self.encode(entry[0]))
+ if entry[4]:
+ # add "extra" info
+ ret.append(r'\sphinxstyleindexextra{%s}' % self.encode(entry[4]))
+ ret.append(r'\sphinxstyleindexpageref{%s:%s}' %
+ (entry[2], self.idescape(entry[3])) + CR)
+ ret.append(r'\end{sphinxtheindex}' + CR)
+
+ ret = []
+ # latex_domain_indices can be False/True or a list of index names
+ if indices_config := self.config.latex_domain_indices:
+ if not isinstance(indices_config, bool):
+ check_names = True
+ indices_config = frozenset(indices_config)
+ else:
+ check_names = False
+ for domain_name in sorted(self.builder.env.domains):
+ domain = self.builder.env.domains[domain_name]
+ for index_cls in domain.indices:
+ index_name = f'{domain.name}-{index_cls.name}'
+ if check_names and index_name not in indices_config:
+ continue
+ content, collapsed = index_cls(domain).generate(
+ self.builder.docnames)
+ if content:
+ ret.append(r'\renewcommand{\indexname}{%s}' % index_cls.localname + CR)
+ generate(content, collapsed)
+
+ return ''.join(ret)
+
+ def render(self, template_name: str, variables: dict[str, Any]) -> str:
+ renderer = LaTeXRenderer(latex_engine=self.config.latex_engine)
+ for template_dir in self.config.templates_path:
+ template = path.join(self.builder.confdir, template_dir,
+ template_name)
+ if path.exists(template):
+ return renderer.render(template, variables)
+ elif template.endswith('.jinja'):
+ legacy_template = template.removesuffix('.jinja') + '_t'
+ if path.exists(legacy_template):
+ logger.warning(__('template %s not found; loading from legacy %s instead'),
+ template_name, legacy_template)
+ return renderer.render(legacy_template, variables)
+
+ return renderer.render(template_name, variables)
+
@property
- def table(self) ->(Table | None):
+ def table(self) -> Table | None:
"""Get current table."""
+ if self.tables:
+ return self.tables[-1]
+ else:
+ return None
+
+ def visit_document(self, node: Element) -> None:
+ self.curfilestack.append(node.get('docname', ''))
+ if self.first_document == 1:
+ # the first document is all the regular content ...
+ self.first_document = 0
+ elif self.first_document == 0:
+ # ... and all others are the appendices
+ self.body.append(CR + r'\appendix' + CR)
+ self.first_document = -1
+ if 'docname' in node:
+ self.body.append(self.hypertarget(':doc'))
+ # "- 1" because the level is increased before the title is visited
+ self.sectionlevel = self.top_sectionlevel - 1
+
+ def depart_document(self, node: Element) -> None:
pass
+
+ def visit_start_of_file(self, node: Element) -> None:
+ self.curfilestack.append(node['docname'])
+ self.body.append(CR + r'\sphinxstepscope' + CR)
+
+ def depart_start_of_file(self, node: Element) -> None:
+ self.curfilestack.pop()
+
+ def visit_section(self, node: Element) -> None:
+ if not self.this_is_the_title:
+ self.sectionlevel += 1
+ self.body.append(BLANKLINE)
+
+ def depart_section(self, node: Element) -> None:
+ self.sectionlevel = max(self.sectionlevel - 1,
+ self.top_sectionlevel - 1)
+
+ def visit_problematic(self, node: Element) -> None:
+ self.body.append(r'{\color{red}\bfseries{}')
+
+ def depart_problematic(self, node: Element) -> None:
+ self.body.append('}')
+
+ def visit_topic(self, node: Element) -> None:
+ self.in_minipage += 1
+ if 'contents' in node.get('classes', []):
+ self.body.append(CR + r'\begin{sphinxcontents}' + CR)
+ self.context.append(r'\end{sphinxcontents}' + CR)
+ else:
+ self.body.append(CR + r'\begin{sphinxtopic}' + CR)
+ self.context.append(r'\end{sphinxtopic}' + CR)
+
+ def depart_topic(self, node: Element) -> None:
+ self.in_minipage -= 1
+ self.body.append(self.context.pop())
+
+ def visit_sidebar(self, node: Element) -> None:
+ self.in_minipage += 1
+ self.body.append(CR + r'\begin{sphinxsidebar}' + CR)
+ self.context.append(r'\end{sphinxsidebar}' + CR)
depart_sidebar = depart_topic
- def _visit_sig_parameter_list(self, node: Element, parameter_group:
- type[Element]) ->None:
+ def visit_glossary(self, node: Element) -> None:
+ pass
+
+ def depart_glossary(self, node: Element) -> None:
+ pass
+
+ def visit_productionlist(self, node: Element) -> None:
+ self.body.append(BLANKLINE)
+ self.body.append(r'\begin{productionlist}' + CR)
+ self.in_production_list = 1
+
+ def depart_productionlist(self, node: Element) -> None:
+ self.body.append(r'\end{productionlist}' + BLANKLINE)
+ self.in_production_list = 0
+
+ def visit_production(self, node: Element) -> None:
+ if node['tokenname']:
+ tn = node['tokenname']
+ self.body.append(self.hypertarget('grammar-token-' + tn))
+ self.body.append(r'\production{%s}{' % self.encode(tn))
+ else:
+ self.body.append(r'\productioncont{')
+
+ def depart_production(self, node: Element) -> None:
+ self.body.append('}' + CR)
+
+ def visit_transition(self, node: Element) -> None:
+ self.body.append(self.elements['transition'])
+
+ def depart_transition(self, node: Element) -> None:
+ pass
+
+ def visit_title(self, node: Element) -> None:
+ parent = node.parent
+ if isinstance(parent, addnodes.seealso):
+ # the environment already handles this
+ raise nodes.SkipNode
+ if isinstance(parent, nodes.section):
+ if self.this_is_the_title:
+ if len(node.children) != 1 and not isinstance(node.children[0],
+ nodes.Text):
+ logger.warning(__('document title is not a single Text node'),
+ location=node)
+ if not self.elements['title']:
+ # text needs to be escaped since it is inserted into
+ # the output literally
+ self.elements['title'] = self.escape(node.astext())
+ self.this_is_the_title = 0
+ raise nodes.SkipNode
+ short = ''
+ if any(node.findall(nodes.image)):
+ short = ('[%s]' % self.escape(' '.join(clean_astext(node).split())))
+
+ try:
+ self.body.append(fr'\{self.sectionnames[self.sectionlevel]}{short}{{')
+ except IndexError:
+ # just use "subparagraph", it's not numbered anyway
+ self.body.append(fr'\{self.sectionnames[-1]}{short}{{')
+ self.context.append('}' + CR + self.hypertarget_to(node.parent))
+ elif isinstance(parent, nodes.topic):
+ if 'contents' in parent.get('classes', []):
+ self.body.append(r'\sphinxstylecontentstitle{')
+ else:
+ self.body.append(r'\sphinxstyletopictitle{')
+ self.context.append('}' + CR)
+ elif isinstance(parent, nodes.sidebar):
+ self.body.append(r'\sphinxstylesidebartitle{')
+ self.context.append('}' + CR)
+ elif isinstance(parent, nodes.Admonition):
+ self.body.append('{')
+ self.context.append('}' + CR)
+ elif isinstance(parent, nodes.table):
+ # Redirect body output until title is finished.
+ self.pushbody([])
+ else:
+ logger.warning(__('encountered title node not in section, topic, table, '
+ 'admonition or sidebar'),
+ location=node)
+ self.body.append(r'\sphinxstyleothertitle{')
+ self.context.append('}' + CR)
+ self.in_title = 1
+
+ def depart_title(self, node: Element) -> None:
+ self.in_title = 0
+ if isinstance(node.parent, nodes.table):
+ assert self.table is not None
+ self.table.caption = self.popbody()
+ else:
+ self.body.append(self.context.pop())
+
+ def visit_subtitle(self, node: Element) -> None:
+ if isinstance(node.parent, nodes.sidebar):
+ self.body.append(r'\sphinxstylesidebarsubtitle{')
+ self.context.append('}' + CR)
+ else:
+ self.context.append('')
+
+ def depart_subtitle(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+
+ #############################################################
+ # Domain-specific object descriptions
+ #############################################################
+
+ # Top-level nodes for descriptions
+ ##################################
+
+ def visit_desc(self, node: Element) -> None:
+ if self.config.latex_show_urls == 'footnote':
+ self.body.append(BLANKLINE)
+ self.body.append(r'\begin{savenotes}\begin{fulllineitems}' + CR)
+ else:
+ self.body.append(BLANKLINE)
+ self.body.append(r'\begin{fulllineitems}' + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_desc(self, node: Element) -> None:
+ if self.in_desc_signature:
+ self.body.append(CR + r'\pysigstopsignatures')
+ self.in_desc_signature = False
+ if self.config.latex_show_urls == 'footnote':
+ self.body.append(CR + r'\end{fulllineitems}\end{savenotes}' + BLANKLINE)
+ else:
+ self.body.append(CR + r'\end{fulllineitems}' + BLANKLINE)
+
+ def _visit_signature_line(self, node: Element) -> None:
+ def next_sibling(e: Node) -> Node | None:
+ try:
+ return e.parent[e.parent.index(e) + 1]
+ except (AttributeError, IndexError):
+ return None
+
+ def has_multi_line(e: Element) -> bool:
+ return e.get('multi_line_parameter_list')
+
+ self.has_tp_list = False
+ self.orphan_tp_list = False
+
+ for child in node:
+ if isinstance(child, addnodes.desc_type_parameter_list):
+ self.has_tp_list = True
+ multi_tp_list = has_multi_line(child)
+ arglist = next_sibling(child)
+ if isinstance(arglist, addnodes.desc_parameterlist):
+ # tp_list + arglist: \macro{name}{tp_list}{arglist}{retann}
+ multi_arglist = has_multi_line(arglist)
+ else:
+ # orphan tp_list: \macro{name}{tp_list}{}{retann}
+ # see: https://github.com/sphinx-doc/sphinx/issues/12543
+ self.orphan_tp_list = True
+ multi_arglist = False
+
+ if multi_tp_list:
+ if multi_arglist:
+ self.body.append(CR + r'\pysigwithonelineperargwithonelinepertparg{')
+ else:
+ self.body.append(CR + r'\pysiglinewithargsretwithonelinepertparg{')
+ else:
+ if multi_arglist:
+ self.body.append(CR + r'\pysigwithonelineperargwithtypelist{')
+ else:
+ self.body.append(CR + r'\pysiglinewithargsretwithtypelist{')
+ break
+
+ if isinstance(child, addnodes.desc_parameterlist):
+ # arglist only: \macro{name}{arglist}{retann}
+ if has_multi_line(child):
+ self.body.append(CR + r'\pysigwithonelineperarg{')
+ else:
+ self.body.append(CR + r'\pysiglinewithargsret{')
+ break
+ else:
+ # no tp_list, no arglist: \macro{name}
+ self.body.append(CR + r'\pysigline{')
+
+ def _depart_signature_line(self, node: Element) -> None:
+ self.body.append('}')
+
+ def visit_desc_signature(self, node: Element) -> None:
+ hyper = ''
+ if node.parent['objtype'] != 'describe' and node['ids']:
+ for id in node['ids']:
+ hyper += self.hypertarget(id)
+ self.body.append(hyper)
+ if not self.in_desc_signature:
+ self.in_desc_signature = True
+ self.body.append(CR + r'\pysigstartsignatures')
+ if not node.get('is_multiline'):
+ self._visit_signature_line(node)
+ else:
+ self.body.append(CR + r'\pysigstartmultiline')
+
+ def depart_desc_signature(self, node: Element) -> None:
+ if not node.get('is_multiline'):
+ self._depart_signature_line(node)
+ else:
+ self.body.append(CR + r'\pysigstopmultiline')
+
+ def visit_desc_signature_line(self, node: Element) -> None:
+ self._visit_signature_line(node)
+
+ def depart_desc_signature_line(self, node: Element) -> None:
+ self._depart_signature_line(node)
+
+ def visit_desc_content(self, node: Element) -> None:
+ assert self.in_desc_signature
+ self.body.append(CR + r'\pysigstopsignatures')
+ self.in_desc_signature = False
+
+ def depart_desc_content(self, node: Element) -> None:
+ pass
+
+ def visit_desc_inline(self, node: Element) -> None:
+ self.body.append(r'\sphinxcode{\sphinxupquote{')
+
+ def depart_desc_inline(self, node: Element) -> None:
+ self.body.append('}}')
+
+ # Nodes for high-level structure in signatures
+ ##############################################
+
+ def visit_desc_name(self, node: Element) -> None:
+ self.body.append(r'\sphinxbfcode{\sphinxupquote{')
+ self.literal_whitespace += 1
+
+ def depart_desc_name(self, node: Element) -> None:
+ self.body.append('}}')
+ self.literal_whitespace -= 1
+
+ def visit_desc_addname(self, node: Element) -> None:
+ self.body.append(r'\sphinxcode{\sphinxupquote{')
+ self.literal_whitespace += 1
+
+ def depart_desc_addname(self, node: Element) -> None:
+ self.body.append('}}')
+ self.literal_whitespace -= 1
+
+ def visit_desc_type(self, node: Element) -> None:
+ pass
+
+ def depart_desc_type(self, node: Element) -> None:
+ pass
+
+ def visit_desc_returns(self, node: Element) -> None:
+ self.body.append(r'{ $\rightarrow$ ')
+
+ def depart_desc_returns(self, node: Element) -> None:
+ self.body.append(r'}')
+
+ def _visit_sig_parameter_list(self, node: Element, parameter_group: type[Element]) -> None:
"""Visit a signature parameters or type parameters list.
The *parameter_group* value is the type of a child node acting as a required parameter
@@ -306,15 +862,765 @@ class LaTeXTranslator(SphinxTranslator):
The caller is responsible for closing adding surrounding LaTeX macro argument start
and stop tokens.
"""
+ self.is_first_param = True
+ self.optional_param_level = 0
+ self.params_left_at_level = 0
+ self.param_group_index = 0
+ # Counts as what we call a parameter group either a required parameter, or a
+ # set of contiguous optional ones.
+ self.list_is_required_param = [isinstance(c, parameter_group) for c in node.children]
+ # How many required parameters are left.
+ self.required_params_left = sum(self.list_is_required_param)
+ self.param_separator = r'\sphinxparamcomma '
+ self.multi_line_parameter_list = node.get('multi_line_parameter_list', False)
+
+ def visit_desc_parameterlist(self, node: Element) -> None:
+ if self.has_tp_list:
+ if self.orphan_tp_list:
+ # close type parameters list (#2)
+ self.body.append('}{')
+ # empty parameters list argument (#3)
+ return
+ else:
+ # close name argument (#1), open parameters list argument (#2)
+ self.body.append('}{')
+ self._visit_sig_parameter_list(node, addnodes.desc_parameter)
+
+ def depart_desc_parameterlist(self, node: Element) -> None:
+ # close parameterlist, open return annotation
+ self.body.append('}{')
+
+ def visit_desc_type_parameter_list(self, node: Element) -> None:
+ # close name argument (#1), open type parameters list argument (#2)
+ self.body.append('}{')
+ self._visit_sig_parameter_list(node, addnodes.desc_type_parameter)
+
+ def depart_desc_type_parameter_list(self, node: Element) -> None:
+ # close type parameters list, open parameters list argument (#3)
+ self.body.append('}{')
+
+ def _visit_sig_parameter(self, node: Element, parameter_macro: str) -> None:
+ if self.is_first_param:
+ self.is_first_param = False
+ elif not self.multi_line_parameter_list and not self.required_params_left:
+ self.body.append(self.param_separator)
+ if self.optional_param_level == 0:
+ self.required_params_left -= 1
+ else:
+ self.params_left_at_level -= 1
+ if not node.hasattr('noemph'):
+ self.body.append(parameter_macro)
+
+ def _depart_sig_parameter(self, node: Element) -> None:
+ if not node.hasattr('noemph'):
+ self.body.append('}')
+ is_required = self.list_is_required_param[self.param_group_index]
+ if self.multi_line_parameter_list:
+ is_last_group = self.param_group_index + 1 == len(self.list_is_required_param)
+ next_is_required = (
+ not is_last_group
+ and self.list_is_required_param[self.param_group_index + 1]
+ )
+ opt_param_left_at_level = self.params_left_at_level > 0
+ if opt_param_left_at_level or is_required and (is_last_group or next_is_required):
+ self.body.append(self.param_separator)
+
+ elif self.required_params_left:
+ self.body.append(self.param_separator)
+
+ if is_required:
+ self.param_group_index += 1
+
+ def visit_desc_parameter(self, node: Element) -> None:
+ self._visit_sig_parameter(node, r'\sphinxparam{')
+
+ def depart_desc_parameter(self, node: Element) -> None:
+ self._depart_sig_parameter(node)
+
+ def visit_desc_type_parameter(self, node: Element) -> None:
+ self._visit_sig_parameter(node, r'\sphinxtypeparam{')
+
+ def depart_desc_type_parameter(self, node: Element) -> None:
+ self._depart_sig_parameter(node)
+
+ def visit_desc_optional(self, node: Element) -> None:
+ self.params_left_at_level = sum(isinstance(c, addnodes.desc_parameter)
+ for c in node.children)
+ self.optional_param_level += 1
+ self.max_optional_param_level = self.optional_param_level
+ if self.multi_line_parameter_list:
+ if self.is_first_param:
+ self.body.append(r'\sphinxoptional{')
+ elif self.required_params_left:
+ self.body.append(self.param_separator)
+ self.body.append(r'\sphinxoptional{')
+ else:
+ self.body.append(r'\sphinxoptional{')
+ self.body.append(self.param_separator)
+ else:
+ self.body.append(r'\sphinxoptional{')
+
+ def depart_desc_optional(self, node: Element) -> None:
+ self.optional_param_level -= 1
+ if self.multi_line_parameter_list:
+ # If it's the first time we go down one level, add the separator before the
+ # bracket.
+ if self.optional_param_level == self.max_optional_param_level - 1:
+ self.body.append(self.param_separator)
+ self.body.append('}')
+ if self.optional_param_level == 0:
+ self.param_group_index += 1
+
+ def visit_desc_annotation(self, node: Element) -> None:
+ self.body.append(r'\sphinxbfcode{\sphinxupquote{')
+
+ def depart_desc_annotation(self, node: Element) -> None:
+ self.body.append('}}')
+
+ ##############################################
+
+ def visit_seealso(self, node: Element) -> None:
+ self.body.append(BLANKLINE)
+ self.body.append(r'\begin{sphinxseealso}{%s:}' % admonitionlabels['seealso'] + CR)
+ self.no_latex_floats += 1
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_seealso(self, node: Element) -> None:
+ self.body.append(BLANKLINE)
+ self.body.append(r'\end{sphinxseealso}')
+ self.body.append(BLANKLINE)
+ self.no_latex_floats -= 1
+
+ def visit_rubric(self, node: nodes.rubric) -> None:
+ if len(node) == 1 and node.astext() in ('Footnotes', _('Footnotes')):
+ raise nodes.SkipNode
+ tag = 'subsubsection'
+ if 'heading-level' in node:
+ level = node['heading-level']
+ try:
+ tag = self.sectionnames[self.top_sectionlevel - 1 + level]
+ except Exception:
+ logger.warning(
+ __('unsupported rubric heading level: %s'),
+ level,
+ type='latex',
+ location=node
+ )
+
+ self.body.append(rf'\{tag}*{{')
+ self.context.append('}' + CR)
+ self.in_title = 1
+
+ def depart_rubric(self, node: nodes.rubric) -> None:
+ self.in_title = 0
+ self.body.append(self.context.pop())
+
+ def visit_footnote(self, node: Element) -> None:
+ self.in_footnote += 1
+ label = cast(nodes.label, node[0])
+ if self.in_parsed_literal:
+ self.body.append(r'\begin{footnote}[%s]' % label.astext())
+ else:
+ self.body.append('%' + CR)
+ self.body.append(r'\begin{footnote}[%s]' % label.astext())
+ if 'referred' in node:
+ # TODO: in future maybe output a latex macro with backrefs here
+ pass
+ self.body.append(r'\sphinxAtStartFootnote' + CR)
+
+ def depart_footnote(self, node: Element) -> None:
+ if self.in_parsed_literal:
+ self.body.append(r'\end{footnote}')
+ else:
+ self.body.append('%' + CR)
+ self.body.append(r'\end{footnote}')
+ self.in_footnote -= 1
+
+ def visit_label(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_tabular_col_spec(self, node: Element) -> None:
+ self.next_table_colspec = node['spec']
+ raise nodes.SkipNode
+
+ def visit_table(self, node: Element) -> None:
+ if len(self.tables) == 1:
+ assert self.table is not None
+ if self.table.get_table_type() == 'longtable':
+ raise UnsupportedError(
+ '%s:%s: longtable does not support nesting a table.' %
+ (self.curfilestack[-1], node.line or ''))
+ # change type of parent table to tabular
+ # see https://groups.google.com/d/msg/sphinx-users/7m3NeOBixeo/9LKP2B4WBQAJ
+ self.table.has_problematic = True
+ elif len(self.tables) > 2:
+ raise UnsupportedError(
+ '%s:%s: deeply nested tables are not implemented.' %
+ (self.curfilestack[-1], node.line or ''))
+
+ table = Table(node)
+ self.tables.append(table)
+ if table.colsep is None:
+ table.colsep = '|' * (
+ 'booktabs' not in self.builder.config.latex_table_style
+ and 'borderless' not in self.builder.config.latex_table_style
+ )
+ if self.next_table_colspec:
+ table.colspec = '{%s}' % self.next_table_colspec + CR
+ if '|' in table.colspec:
+ table.styles.append('vlines')
+ table.colsep = '|'
+ else:
+ table.styles.append('novlines')
+ table.colsep = ''
+ if 'colwidths-given' in node.get('classes', []):
+ logger.info(__('both tabularcolumns and :widths: option are given. '
+ ':widths: is ignored.'), location=node)
+ self.next_table_colspec = None
+
+ def depart_table(self, node: Element) -> None:
+ assert self.table is not None
+ labels = self.hypertarget_to(node)
+ table_type = self.table.get_table_type()
+ table = self.render(table_type + '.tex.jinja',
+ {'table': self.table, 'labels': labels})
+ self.body.append(BLANKLINE)
+ self.body.append(table)
+ self.body.append(CR)
+
+ self.tables.pop()
+
+ def visit_colspec(self, node: Element) -> None:
+ assert self.table is not None
+ self.table.colcount += 1
+ if 'colwidth' in node:
+ self.table.colwidths.append(node['colwidth'])
+ if 'stub' in node:
+ self.table.stubs.append(self.table.colcount - 1)
+
+ def depart_colspec(self, node: Element) -> None:
+ pass
+
+ def visit_tgroup(self, node: Element) -> None:
+ pass
+
+ def depart_tgroup(self, node: Element) -> None:
+ pass
+
+ def visit_thead(self, node: Element) -> None:
+ assert self.table is not None
+ # Redirect head output until header is finished.
+ self.pushbody(self.table.header)
+
+ def depart_thead(self, node: Element) -> None:
+ if self.body and self.body[-1] == r'\sphinxhline':
+ self.body.pop()
+ self.popbody()
+
+ def visit_tbody(self, node: Element) -> None:
+ assert self.table is not None
+ # Redirect body output until table is finished.
+ self.pushbody(self.table.body)
+
+ def depart_tbody(self, node: Element) -> None:
+ if self.body and self.body[-1] == r'\sphinxhline':
+ self.body.pop()
+ self.popbody()
+
+ def visit_row(self, node: Element) -> None:
+ assert self.table is not None
+ self.table.col = 0
+ _colsep = self.table.colsep
+ # fill columns if the row starts with the bottom of multirow cell
+ while True:
+ cell = self.table.cell(self.table.row, self.table.col)
+ if cell is None: # not a bottom of multirow cell
+ break
+ # a bottom of multirow cell
+ self.table.col += cell.width
+ if cell.col:
+ self.body.append('&')
+ if cell.width == 1:
+ # insert suitable strut for equalizing row heights in given multirow
+ self.body.append(r'\sphinxtablestrut{%d}' % cell.cell_id)
+ else: # use \multicolumn for wide multirow cell
+ self.body.append(r'\multicolumn{%d}{%sl%s}{\sphinxtablestrut{%d}}' %
+ (cell.width, _colsep, _colsep, cell.cell_id))
+
+ def depart_row(self, node: Element) -> None:
+ assert self.table is not None
+ self.body.append(r'\\' + CR)
+ cells = [self.table.cell(self.table.row, i) for i in range(self.table.colcount)]
+ underlined = [cell.row + cell.height == self.table.row + 1 # type: ignore[union-attr]
+ for cell in cells]
+ if all(underlined):
+ self.body.append(r'\sphinxhline')
+ else:
+ i = 0
+ underlined.extend([False]) # sentinel
+ if underlined[0] is False:
+ i = 1
+ while i < self.table.colcount and underlined[i] is False:
+ if cells[i - 1].cell_id != cells[i].cell_id: # type: ignore[union-attr]
+ self.body.append(r'\sphinxvlinecrossing{%d}' % i)
+ i += 1
+ while i < self.table.colcount:
+ # each time here underlined[i] is True
+ j = underlined[i:].index(False)
+ self.body.append(r'\sphinxcline{%d-%d}' % (i + 1, i + j))
+ i += j
+ i += 1
+ while i < self.table.colcount and underlined[i] is False:
+ if cells[i - 1].cell_id != cells[i].cell_id: # type: ignore[union-attr]
+ self.body.append(r'\sphinxvlinecrossing{%d}' % i)
+ i += 1
+ self.body.append(r'\sphinxfixclines{%d}' % self.table.colcount)
+ self.table.row += 1
+
+ def visit_entry(self, node: Element) -> None:
+ assert self.table is not None
+ if self.table.col > 0:
+ self.body.append('&')
+ self.table.add_cell(node.get('morerows', 0) + 1, node.get('morecols', 0) + 1)
+ cell = self.table.cell()
+ assert cell is not None
+ context = ''
+ _colsep = self.table.colsep
+ if cell.width > 1:
+ if self.config.latex_use_latex_multicolumn:
+ if self.table.col == 0:
+ self.body.append(r'\multicolumn{%d}{%sl%s}{%%' %
+ (cell.width, _colsep, _colsep) + CR)
+ else:
+ self.body.append(r'\multicolumn{%d}{l%s}{%%' % (cell.width, _colsep) + CR)
+ context = '}%' + CR
+ else:
+ self.body.append(r'\sphinxstartmulticolumn{%d}%%' % cell.width + CR)
+ context = r'\sphinxstopmulticolumn' + CR
+ if cell.height > 1:
+ # \sphinxmultirow 2nd arg "cell_id" will serve as id for LaTeX macros as well
+ self.body.append(r'\sphinxmultirow{%d}{%d}{%%' % (cell.height, cell.cell_id) + CR)
+ context = '}%' + CR + context
+ if cell.width > 1 or cell.height > 1:
+ self.body.append(r'\begin{varwidth}[t]{\sphinxcolwidth{%d}{%d}}'
+ % (cell.width, self.table.colcount) + CR)
+ context = (r'\par' + CR + r'\vskip-\baselineskip'
+ r'\vbox{\hbox{\strut}}\end{varwidth}%' + CR + context)
+ self.needs_linetrimming = 1
+ if len(list(node.findall(nodes.paragraph))) >= 2:
+ self.table.has_oldproblematic = True
+ if isinstance(node.parent.parent, nodes.thead) or (cell.col in self.table.stubs):
+ if len(node) == 1 and isinstance(node[0], nodes.paragraph) and node.astext() == '':
+ pass
+ else:
+ self.body.append(r'\sphinxstyletheadfamily ')
+ if self.needs_linetrimming:
+ self.pushbody([])
+ self.context.append(context)
+
+ def depart_entry(self, node: Element) -> None:
+ if self.needs_linetrimming:
+ self.needs_linetrimming = 0
+ body = self.popbody()
+
+ # Remove empty lines from top of merged cell
+ while body and body[0] == CR:
+ body.pop(0)
+ self.body.extend(body)
+
+ self.body.append(self.context.pop())
+
+ assert self.table is not None
+ cell = self.table.cell()
+ assert cell is not None
+ self.table.col += cell.width
+ _colsep = self.table.colsep
+
+ # fill columns if next ones are a bottom of wide-multirow cell
+ while True:
+ nextcell = self.table.cell()
+ if nextcell is None: # not a bottom of multirow cell
+ break
+ # a bottom part of multirow cell
+ self.body.append('&')
+ if nextcell.width == 1:
+ # insert suitable strut for equalizing row heights in multirow
+ # they also serve to clear colour panels which would hide the text
+ self.body.append(r'\sphinxtablestrut{%d}' % nextcell.cell_id)
+ else:
+ # use \multicolumn for not first row of wide multirow cell
+ self.body.append(r'\multicolumn{%d}{l%s}{\sphinxtablestrut{%d}}' %
+ (nextcell.width, _colsep, nextcell.cell_id))
+ self.table.col += nextcell.width
+
+ def visit_acks(self, node: Element) -> None:
+ # this is a list in the source, but should be rendered as a
+ # comma-separated list here
+ bullet_list = cast(nodes.bullet_list, node[0])
+ list_items = cast(Iterable[nodes.list_item], bullet_list)
+ self.body.append(BLANKLINE)
+ self.body.append(', '.join(n.astext() for n in list_items) + '.')
+ self.body.append(BLANKLINE)
+ raise nodes.SkipNode
+
+ def visit_bullet_list(self, node: Element) -> None:
+ if not self.compact_list:
+ self.body.append(r'\begin{itemize}' + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_bullet_list(self, node: Element) -> None:
+ if not self.compact_list:
+ self.body.append(r'\end{itemize}' + CR)
+
+ def visit_enumerated_list(self, node: Element) -> None:
+ def get_enumtype(node: Element) -> str:
+ enumtype = node.get('enumtype', 'arabic')
+ if 'alpha' in enumtype and (node.get('start', 0) + len(node)) > 26:
+ # fallback to arabic if alphabet counter overflows
+ enumtype = 'arabic'
+
+ return enumtype
+
+ def get_nested_level(node: Element) -> int:
+ if node is None:
+ return 0
+ elif isinstance(node, nodes.enumerated_list):
+ return get_nested_level(node.parent) + 1
+ else:
+ return get_nested_level(node.parent)
+
+ enum = "enum%s" % toRoman(get_nested_level(node)).lower()
+ enumnext = "enum%s" % toRoman(get_nested_level(node) + 1).lower()
+ style = ENUMERATE_LIST_STYLE.get(get_enumtype(node))
+ prefix = node.get('prefix', '')
+ suffix = node.get('suffix', '.')
+
+ self.body.append(r'\begin{enumerate}' + CR)
+ self.body.append(r'\sphinxsetlistlabels{%s}{%s}{%s}{%s}{%s}%%' %
+ (style, enum, enumnext, prefix, suffix) + CR)
+ if 'start' in node:
+ self.body.append(r'\setcounter{%s}{%d}' % (enum, node['start'] - 1) + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_enumerated_list(self, node: Element) -> None:
+ self.body.append(r'\end{enumerate}' + CR)
+
+ def visit_list_item(self, node: Element) -> None:
+ # Append "{}" in case the next character is "[", which would break
+ # LaTeX's list environment (no numbering and the "[" is not printed).
+ self.body.append(r'\item {} ')
+
+ def depart_list_item(self, node: Element) -> None:
+ self.body.append(CR)
+
+ def visit_definition_list(self, node: Element) -> None:
+ self.body.append(r'\begin{description}' + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_definition_list(self, node: Element) -> None:
+ self.body.append(r'\end{description}' + CR)
+
+ def visit_definition_list_item(self, node: Element) -> None:
+ pass
+
+ def depart_definition_list_item(self, node: Element) -> None:
+ pass
+
+ def visit_term(self, node: Element) -> None:
+ self.in_term += 1
+ ctx = ''
+ if node.get('ids'):
+ ctx = r'\phantomsection'
+ for node_id in node['ids']:
+ ctx += self.hypertarget(node_id, anchor=False)
+ ctx += r'}'
+ self.body.append(r'\sphinxlineitem{')
+ self.context.append(ctx)
+
+ def depart_term(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+ self.in_term -= 1
+
+ def visit_classifier(self, node: Element) -> None:
+ self.body.append('{[}')
+
+ def depart_classifier(self, node: Element) -> None:
+ self.body.append('{]}')
+
+ def visit_definition(self, node: Element) -> None:
+ pass
+
+ def depart_definition(self, node: Element) -> None:
+ self.body.append(CR)
+
+ def visit_field_list(self, node: Element) -> None:
+ self.body.append(r'\begin{quote}\begin{description}' + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_field_list(self, node: Element) -> None:
+ self.body.append(r'\end{description}\end{quote}' + CR)
+
+ def visit_field(self, node: Element) -> None:
+ pass
+
+ def depart_field(self, node: Element) -> None:
pass
+
visit_field_name = visit_term
depart_field_name = depart_term
+
visit_field_body = visit_definition
depart_field_body = depart_definition
- def is_inline(self, node: Element) ->bool:
+ def visit_paragraph(self, node: Element) -> None:
+ index = node.parent.index(node)
+ if (index > 0 and isinstance(node.parent, nodes.compound) and
+ not isinstance(node.parent[index - 1], nodes.paragraph) and
+ not isinstance(node.parent[index - 1], nodes.compound)):
+ # insert blank line, if the paragraph follows a non-paragraph node in a compound
+ self.body.append(r'\noindent' + CR)
+ elif index == 1 and isinstance(node.parent, nodes.footnote | footnotetext):
+ # don't insert blank line, if the paragraph is second child of a footnote
+ # (first one is label node)
+ pass
+ else:
+ # the \sphinxAtStartPar is to allow hyphenation of first word of
+ # a paragraph in narrow contexts such as in a table cell
+ # added as two items (cf. line trimming in depart_entry())
+ self.body.extend([CR, r'\sphinxAtStartPar' + CR])
+
+ def depart_paragraph(self, node: Element) -> None:
+ self.body.append(CR)
+
+ def visit_centered(self, node: Element) -> None:
+ self.body.append(CR + r'\begin{center}')
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_centered(self, node: Element) -> None:
+ self.body.append(CR + r'\end{center}')
+
+ def visit_hlist(self, node: Element) -> None:
+ self.compact_list += 1
+ ncolumns = node['ncolumns']
+ if self.compact_list > 1:
+ self.body.append(r'\setlength{\multicolsep}{0pt}' + CR)
+ self.body.append(r'\begin{multicols}{' + ncolumns + r'}\raggedright' + CR)
+ self.body.append(r'\begin{itemize}\setlength{\itemsep}{0pt}'
+ r'\setlength{\parskip}{0pt}' + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_hlist(self, node: Element) -> None:
+ self.compact_list -= 1
+ self.body.append(r'\end{itemize}\raggedcolumns\end{multicols}' + CR)
+
+ def visit_hlistcol(self, node: Element) -> None:
+ pass
+
+ def depart_hlistcol(self, node: Element) -> None:
+ # \columnbreak would guarantee same columns as in html output. But
+ # some testing with long items showed that columns may be too uneven.
+ # And in case only of short items, the automatic column breaks should
+ # match the ones pre-computed by the hlist() directive.
+ # self.body.append(r'\columnbreak\n')
+ pass
+
+ def latex_image_length(self, width_str: str, scale: int = 100) -> str | None:
+ try:
+ return rstdim_to_latexdim(width_str, scale)
+ except ValueError:
+ logger.warning(__('dimension unit %s is invalid. Ignored.'), width_str)
+ return None
+
+ def is_inline(self, node: Element) -> bool:
"""Check whether a node represents an inline element."""
+ return isinstance(node.parent, nodes.TextElement)
+
+ def visit_image(self, node: Element) -> None:
+ pre: list[str] = [] # in reverse order
+ post: list[str] = []
+ include_graphics_options = []
+ has_hyperlink = isinstance(node.parent, nodes.reference)
+ if has_hyperlink:
+ is_inline = self.is_inline(node.parent)
+ else:
+ is_inline = self.is_inline(node)
+ if 'width' in node:
+ if 'scale' in node:
+ w = self.latex_image_length(node['width'], node['scale'])
+ else:
+ w = self.latex_image_length(node['width'])
+ if w:
+ include_graphics_options.append('width=%s' % w)
+ if 'height' in node:
+ if 'scale' in node:
+ h = self.latex_image_length(node['height'], node['scale'])
+ else:
+ h = self.latex_image_length(node['height'])
+ if h:
+ include_graphics_options.append('height=%s' % h)
+ if 'scale' in node:
+ if not include_graphics_options:
+ # if no "width" nor "height", \sphinxincludegraphics will fit
+ # to the available text width if oversized after rescaling.
+ include_graphics_options.append('scale=%s'
+ % (float(node['scale']) / 100.0))
+ if 'align' in node:
+ align_prepost = {
+ # By default latex aligns the top of an image.
+ (1, 'top'): ('', ''),
+ (1, 'middle'): (r'\raisebox{-0.5\height}{', '}'),
+ (1, 'bottom'): (r'\raisebox{-\height}{', '}'),
+ (0, 'center'): (r'{\hspace*{\fill}', r'\hspace*{\fill}}'),
+ # These 2 don't exactly do the right thing. The image should
+ # be floated alongside the paragraph. See
+ # https://www.w3.org/TR/html4/struct/objects.html#adef-align-IMG
+ (0, 'left'): ('{', r'\hspace*{\fill}}'),
+ (0, 'right'): (r'{\hspace*{\fill}', '}'),
+ }
+ try:
+ pre.append(align_prepost[is_inline, node['align']][0])
+ post.append(align_prepost[is_inline, node['align']][1])
+ except KeyError:
+ pass
+ if self.in_parsed_literal:
+ pre.append(r'{\sphinxunactivateextrasandspace ')
+ post.append('}')
+ if not is_inline and not has_hyperlink:
+ pre.append(CR + r'\noindent')
+ post.append(CR)
+ pre.reverse()
+ if node['uri'] in self.builder.images:
+ uri = self.builder.images[node['uri']]
+ else:
+ # missing image!
+ if self.ignore_missing_images:
+ return
+ uri = node['uri']
+ if uri.find('://') != -1:
+ # ignore remote images
+ return
+ self.body.extend(pre)
+ options = ''
+ if include_graphics_options:
+ options = '[%s]' % ','.join(include_graphics_options)
+ base, ext = path.splitext(uri)
+
+ if self.in_title and base:
+ # Lowercase tokens forcely because some fncychap themes capitalize
+ # the options of \sphinxincludegraphics unexpectedly (ex. WIDTH=...).
+ cmd = fr'\lowercase{{\sphinxincludegraphics{options}}}{{{{{base}}}{ext}}}'
+ else:
+ cmd = fr'\sphinxincludegraphics{options}{{{{{base}}}{ext}}}'
+ # escape filepath for includegraphics, https://tex.stackexchange.com/a/202714/41112
+ if '#' in base:
+ cmd = r'{\catcode`\#=12' + cmd + '}'
+ self.body.append(cmd)
+ self.body.extend(post)
+
+ def depart_image(self, node: Element) -> None:
pass
+
+ def visit_figure(self, node: Element) -> None:
+ align = self.elements['figure_align']
+ if self.no_latex_floats:
+ align = "H"
+ if self.table:
+ # Blank line is needed if text precedes
+ self.body.append(BLANKLINE)
+ # TODO: support align option
+ if 'width' in node:
+ length = self.latex_image_length(node['width'])
+ if length:
+ self.body.append(r'\begin{sphinxfigure-in-table}[%s]' % length + CR)
+ self.body.append(r'\centering' + CR)
+ else:
+ self.body.append(r'\begin{sphinxfigure-in-table}' + CR)
+ self.body.append(r'\centering' + CR)
+ if any(isinstance(child, nodes.caption) for child in node):
+ self.body.append(r'\capstart')
+ self.context.append(r'\end{sphinxfigure-in-table}\relax' + CR)
+ elif node.get('align', '') in ('left', 'right'):
+ length = None
+ if 'width' in node:
+ length = self.latex_image_length(node['width'])
+ elif isinstance(node[0], nodes.image) and 'width' in node[0]:
+ length = self.latex_image_length(node[0]['width'])
+ # Insert a blank line to prevent an infinite loop
+ # https://github.com/sphinx-doc/sphinx/issues/7059
+ self.body.append(BLANKLINE)
+ self.body.append(r'\begin{wrapfigure}{%s}{%s}' %
+ ('r' if node['align'] == 'right' else 'l', length or '0pt') + CR)
+ self.body.append(r'\centering')
+ self.context.append(r'\end{wrapfigure}' +
+ BLANKLINE +
+ r'\mbox{}\par\vskip-\dimexpr\baselineskip+\parskip\relax' +
+ CR) # avoid disappearance if no text next issues/11079
+ elif self.in_minipage:
+ self.body.append(CR + r'\begin{center}')
+ self.context.append(r'\end{center}' + CR)
+ else:
+ self.body.append(CR + r'\begin{figure}[%s]' % align + CR)
+ self.body.append(r'\centering' + CR)
+ if any(isinstance(child, nodes.caption) for child in node):
+ self.body.append(r'\capstart' + CR)
+ self.context.append(r'\end{figure}' + CR)
+
+ def depart_figure(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+
+ def visit_caption(self, node: Element) -> None:
+ self.in_caption += 1
+ if isinstance(node.parent, captioned_literal_block):
+ self.body.append(r'\sphinxSetupCaptionForVerbatim{')
+ elif self.in_minipage and isinstance(node.parent, nodes.figure):
+ self.body.append(r'\captionof{figure}{')
+ elif self.table and node.parent.tagname == 'figure':
+ self.body.append(r'\sphinxfigcaption{')
+ else:
+ self.body.append(r'\caption{')
+
+ def depart_caption(self, node: Element) -> None:
+ self.body.append('}')
+ if isinstance(node.parent, nodes.figure):
+ labels = self.hypertarget_to(node.parent)
+ self.body.append(labels)
+ self.in_caption -= 1
+
+ def visit_legend(self, node: Element) -> None:
+ self.body.append(CR + r'\begin{sphinxlegend}')
+
+ def depart_legend(self, node: Element) -> None:
+ self.body.append(r'\end{sphinxlegend}' + CR)
+
+ def visit_admonition(self, node: Element) -> None:
+ self.body.append(CR + r'\begin{sphinxadmonition}{note}')
+ self.no_latex_floats += 1
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_admonition(self, node: Element) -> None:
+ self.body.append(r'\end{sphinxadmonition}' + CR)
+ self.no_latex_floats -= 1
+
+ def _visit_named_admonition(self, node: Element) -> None:
+ label = admonitionlabels[node.tagname]
+ self.body.append(CR + r'\begin{sphinxadmonition}{%s}{%s:}' %
+ (node.tagname, label))
+ self.no_latex_floats += 1
+ if self.table:
+ self.table.has_problematic = True
+
+ def _depart_named_admonition(self, node: Element) -> None:
+ self.body.append(r'\end{sphinxadmonition}' + CR)
+ self.no_latex_floats -= 1
+
visit_attention = _visit_named_admonition
depart_attention = _depart_named_admonition
visit_caution = _visit_named_admonition
@@ -333,12 +1639,684 @@ class LaTeXTranslator(SphinxTranslator):
depart_tip = _depart_named_admonition
visit_warning = _visit_named_admonition
depart_warning = _depart_named_admonition
+
+ def visit_versionmodified(self, node: Element) -> None:
+ pass
+
+ def depart_versionmodified(self, node: Element) -> None:
+ pass
+
+ def visit_target(self, node: Element) -> None:
+ def add_target(id: str) -> None:
+ # indexing uses standard LaTeX index markup, so the targets
+ # will be generated differently
+ if id.startswith('index-'):
+ return
+
+ # equations also need no extra blank line nor hypertarget
+ # TODO: fix this dependency on mathbase extension internals
+ if id.startswith('equation-'):
+ return
+
+ # insert blank line, if the target follows a paragraph node
+ index = node.parent.index(node)
+ if index > 0 and isinstance(node.parent[index - 1], nodes.paragraph):
+ self.body.append(CR)
+
+ # do not generate \phantomsection in \section{}
+ anchor = not self.in_title
+ self.body.append(self.hypertarget(id, anchor=anchor))
+
+ # skip if visitor for next node supports hyperlink
+ next_node: Node = node
+ while isinstance(next_node, nodes.target):
+ next_node = next_node.next_node(ascend=True)
+
+ domain = cast(StandardDomain, self.builder.env.get_domain('std'))
+ if isinstance(next_node, HYPERLINK_SUPPORT_NODES):
+ return
+ if domain.get_enumerable_node_type(next_node) and domain.get_numfig_title(next_node):
+ return
+
+ if 'refuri' in node:
+ return
+ if 'anonymous' in node:
+ return
+ if node.get('refid'):
+ prev_node = get_prev_node(node)
+ if isinstance(prev_node, nodes.reference) and node['refid'] == prev_node['refid']:
+ # a target for a hyperlink reference having alias
+ pass
+ else:
+ add_target(node['refid'])
+ # Temporary fix for https://github.com/sphinx-doc/sphinx/issues/11093
+ # TODO: investigate if a more elegant solution exists (see comments of #11093)
+ if node.get('ismod', False):
+ # Detect if the previous nodes are label targets. If so, remove
+ # the refid thereof from node['ids'] to avoid duplicated ids.
+ def has_dup_label(sib: Node | None) -> bool:
+ return isinstance(sib, nodes.target) and sib.get('refid') in node['ids']
+
+ prev = get_prev_node(node)
+ if has_dup_label(prev):
+ ids = node['ids'][:] # copy to avoid side-effects
+ while has_dup_label(prev):
+ ids.remove(prev['refid']) # type: ignore[index]
+ prev = get_prev_node(prev) # type: ignore[arg-type]
+ else:
+ ids = iter(node['ids']) # read-only iterator
+ else:
+ ids = iter(node['ids']) # read-only iterator
+
+ for id in ids:
+ add_target(id)
+
+ def depart_target(self, node: Element) -> None:
+ pass
+
+ def visit_attribution(self, node: Element) -> None:
+ self.body.append(CR + r'\begin{flushright}' + CR)
+ self.body.append('---')
+
+ def depart_attribution(self, node: Element) -> None:
+ self.body.append(CR + r'\end{flushright}' + CR)
+
+ def visit_index(self, node: Element) -> None:
+ def escape(value: str) -> str:
+ value = self.encode(value)
+ value = value.replace(r'\{', r'\sphinxleftcurlybrace{}')
+ value = value.replace(r'\}', r'\sphinxrightcurlybrace{}')
+ value = value.replace('"', '""')
+ value = value.replace('@', '"@')
+ value = value.replace('!', '"!')
+ value = value.replace('|', r'\textbar{}')
+ return value
+
+ def style(string: str) -> str:
+ match = EXTRA_RE.match(string)
+ if match:
+ return match.expand(r'\\spxentry{\1}\\spxextra{\2}')
+ else:
+ return r'\spxentry{%s}' % string
+
+ if not node.get('inline', True):
+ self.body.append(CR)
+ entries = node['entries']
+ for type, string, _tid, ismain, _key in entries:
+ m = ''
+ if ismain:
+ m = '|spxpagem'
+ try:
+ parts = tuple(map(escape, split_index_msg(type, string)))
+ styled = tuple(map(style, parts))
+ if type == 'single':
+ try:
+ p1, p2 = parts
+ P1, P2 = styled
+ self.body.append(fr'\index{{{p1}@{P1}!{p2}@{P2}{m}}}')
+ except ValueError:
+ p, = parts
+ P, = styled
+ self.body.append(fr'\index{{{p}@{P}{m}}}')
+ elif type == 'pair':
+ p1, p2 = parts
+ P1, P2 = styled
+ self.body.append(fr'\index{{{p1}@{P1}!{p2}@{P2}{m}}}'
+ fr'\index{{{p2}@{P2}!{p1}@{P1}{m}}}')
+ elif type == 'triple':
+ p1, p2, p3 = parts
+ P1, P2, P3 = styled
+ self.body.append(
+ fr'\index{{{p1}@{P1}!{p2} {p3}@{P2} {P3}{m}}}'
+ fr'\index{{{p2}@{P2}!{p3}, {p1}@{P3}, {P1}{m}}}'
+ fr'\index{{{p3}@{P3}!{p1} {p2}@{P1} {P2}{m}}}')
+ elif type in {'see', 'seealso'}:
+ p1, p2 = parts
+ P1, _P2 = styled
+ self.body.append(fr'\index{{{p1}@{P1}|see{{{p2}}}}}')
+ else:
+ logger.warning(__('unknown index entry type %s found'), type)
+ except ValueError as err:
+ logger.warning(str(err))
+ if not node.get('inline', True):
+ self.body.append(r'\ignorespaces ')
+ raise nodes.SkipNode
+
+ def visit_raw(self, node: Element) -> None:
+ if not self.is_inline(node):
+ self.body.append(CR)
+ if 'latex' in node.get('format', '').split():
+ self.body.append(node.astext())
+ if not self.is_inline(node):
+ self.body.append(CR)
+ raise nodes.SkipNode
+
+ def visit_reference(self, node: Element) -> None:
+ if not self.in_title:
+ for id in node.get('ids'):
+ anchor = not self.in_caption
+ self.body += self.hypertarget(id, anchor=anchor)
+ if not self.is_inline(node):
+ self.body.append(CR)
+ uri = node.get('refuri', '')
+ if not uri and node.get('refid'):
+ uri = '%' + self.curfilestack[-1] + '#' + node['refid']
+ if self.in_title or not uri:
+ self.context.append('')
+ elif uri.startswith('#'):
+ # references to labels in the same document
+ id = self.curfilestack[-1] + ':' + uri[1:]
+ self.body.append(self.hyperlink(id))
+ self.body.append(r'\sphinxsamedocref{')
+ if self.config.latex_show_pagerefs and not \
+ self.in_production_list:
+ self.context.append('}}} (%s)' % self.hyperpageref(id))
+ else:
+ self.context.append('}}}')
+ elif uri.startswith('%'):
+ # references to documents or labels inside documents
+ hashindex = uri.find('#')
+ if hashindex == -1:
+ # reference to the document
+ id = uri[1:] + '::doc'
+ else:
+ # reference to a label
+ id = uri[1:].replace('#', ':')
+ self.body.append(self.hyperlink(id))
+ if (len(node) and
+ isinstance(node[0], nodes.Element) and
+ 'std-term' in node[0].get('classes', [])):
+ # don't add a pageref for glossary terms
+ self.context.append('}}}')
+ # mark up as termreference
+ self.body.append(r'\sphinxtermref{')
+ else:
+ self.body.append(r'\sphinxcrossref{')
+ if self.config.latex_show_pagerefs and not self.in_production_list:
+ self.context.append('}}} (%s)' % self.hyperpageref(id))
+ else:
+ self.context.append('}}}')
+ else:
+ if len(node) == 1 and uri == node[0]:
+ if node.get('nolinkurl'):
+ self.body.append(r'\sphinxnolinkurl{%s}' % self.encode_uri(uri))
+ else:
+ self.body.append(r'\sphinxurl{%s}' % self.encode_uri(uri))
+ raise nodes.SkipNode
+ else:
+ self.body.append(r'\sphinxhref{%s}{' % self.encode_uri(uri))
+ self.context.append('}')
+
+ def depart_reference(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+ if not self.is_inline(node):
+ self.body.append(CR)
+
+ def visit_number_reference(self, node: Element) -> None:
+ if node.get('refid'):
+ id = self.curfilestack[-1] + ':' + node['refid']
+ else:
+ id = node.get('refuri', '')[1:].replace('#', ':')
+
+ title = self.escape(node.get('title', '%s')).replace(r'\%s', '%s')
+ if r'\{name\}' in title or r'\{number\}' in title:
+ # new style format (cf. "Fig.%{number}")
+ title = title.replace(r'\{name\}', '{name}').replace(r'\{number\}', '{number}')
+ text = escape_abbr(title).format(name=r'\nameref{%s}' % self.idescape(id),
+ number=r'\ref{%s}' % self.idescape(id))
+ else:
+ # old style format (cf. "Fig.%{number}")
+ text = escape_abbr(title) % (r'\ref{%s}' % self.idescape(id))
+ hyperref = fr'\hyperref[{self.idescape(id)}]{{{text}}}'
+ self.body.append(hyperref)
+
+ raise nodes.SkipNode
+
+ def visit_download_reference(self, node: Element) -> None:
+ pass
+
+ def depart_download_reference(self, node: Element) -> None:
+ pass
+
+ def visit_pending_xref(self, node: Element) -> None:
+ pass
+
+ def depart_pending_xref(self, node: Element) -> None:
+ pass
+
+ def visit_emphasis(self, node: Element) -> None:
+ self.body.append(r'\sphinxstyleemphasis{')
+
+ def depart_emphasis(self, node: Element) -> None:
+ self.body.append('}')
+
+ def visit_literal_emphasis(self, node: Element) -> None:
+ self.body.append(r'\sphinxstyleliteralemphasis{\sphinxupquote{')
+
+ def depart_literal_emphasis(self, node: Element) -> None:
+ self.body.append('}}')
+
+ def visit_strong(self, node: Element) -> None:
+ self.body.append(r'\sphinxstylestrong{')
+
+ def depart_strong(self, node: Element) -> None:
+ self.body.append('}')
+
+ def visit_literal_strong(self, node: Element) -> None:
+ self.body.append(r'\sphinxstyleliteralstrong{\sphinxupquote{')
+
+ def depart_literal_strong(self, node: Element) -> None:
+ self.body.append('}}')
+
+ def visit_abbreviation(self, node: Element) -> None:
+ abbr = node.astext()
+ self.body.append(r'\sphinxstyleabbreviation{')
+ # spell out the explanation once
+ if node.hasattr('explanation') and abbr not in self.handled_abbrs:
+ self.context.append('} (%s)' % self.encode(node['explanation']))
+ self.handled_abbrs.add(abbr)
+ else:
+ self.context.append('}')
+
+ def depart_abbreviation(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+
+ def visit_manpage(self, node: Element) -> None:
+ return self.visit_literal_emphasis(node)
+
+ def depart_manpage(self, node: Element) -> None:
+ return self.depart_literal_emphasis(node)
+
+ def visit_title_reference(self, node: Element) -> None:
+ self.body.append(r'\sphinxtitleref{')
+
+ def depart_title_reference(self, node: Element) -> None:
+ self.body.append('}')
+
+ def visit_thebibliography(self, node: Element) -> None:
+ citations = cast(Iterable[nodes.citation], node)
+ labels = (cast(nodes.label, citation[0]) for citation in citations)
+ longest_label = max((label.astext() for label in labels), key=len)
+ if len(longest_label) > MAX_CITATION_LABEL_LENGTH:
+ # adjust max width of citation labels not to break the layout
+ longest_label = longest_label[:MAX_CITATION_LABEL_LENGTH]
+
+ self.body.append(CR + r'\begin{sphinxthebibliography}{%s}' %
+ self.encode(longest_label) + CR)
+
+ def depart_thebibliography(self, node: Element) -> None:
+ self.body.append(r'\end{sphinxthebibliography}' + CR)
+
+ def visit_citation(self, node: Element) -> None:
+ label = cast(nodes.label, node[0])
+ self.body.append(fr'\bibitem[{self.encode(label.astext())}]'
+ fr'{{{node["docname"]}:{node["ids"][0]}}}')
+
+ def depart_citation(self, node: Element) -> None:
+ pass
+
+ def visit_citation_reference(self, node: Element) -> None:
+ if self.in_title:
+ pass
+ else:
+ self.body.append(fr'\sphinxcite{{{node["docname"]}:{node["refname"]}}}')
+ raise nodes.SkipNode
+
+ def depart_citation_reference(self, node: Element) -> None:
+ pass
+
+ def visit_literal(self, node: Element) -> None:
+ if self.in_title:
+ self.body.append(r'\sphinxstyleliteralintitle{\sphinxupquote{')
+ return
+ elif 'kbd' in node['classes']:
+ self.body.append(r'\sphinxkeyboard{\sphinxupquote{')
+ return
+ lang = node.get("language", None)
+ if 'code' not in node['classes'] or not lang:
+ self.body.append(r'\sphinxcode{\sphinxupquote{')
+ return
+
+ opts = self.config.highlight_options.get(lang, {})
+ hlcode = self.highlighter.highlight_block(
+ node.astext(), lang, opts=opts, location=node, nowrap=True)
+ self.body.append(r'\sphinxcode{\sphinxupquote{%' + CR
+ + hlcode.rstrip() + '%' + CR
+ + '}}')
+ raise nodes.SkipNode
+
+ def depart_literal(self, node: Element) -> None:
+ self.body.append('}}')
+
+ def visit_footnote_reference(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_footnotemark(self, node: Element) -> None:
+ self.body.append(r'\sphinxfootnotemark[')
+
+ def depart_footnotemark(self, node: Element) -> None:
+ self.body.append(']')
+
+ def visit_footnotetext(self, node: Element) -> None:
+ label = cast(nodes.label, node[0])
+ self.body.append('%' + CR)
+ self.body.append(r'\begin{footnotetext}[%s]' % label.astext())
+ self.body.append(r'\sphinxAtStartFootnote' + CR)
+
+ def depart_footnotetext(self, node: Element) -> None:
+ # the \ignorespaces in particular for after table header use
+ self.body.append('%' + CR)
+ self.body.append(r'\end{footnotetext}\ignorespaces ')
+
+ def visit_captioned_literal_block(self, node: Element) -> None:
+ pass
+
+ def depart_captioned_literal_block(self, node: Element) -> None:
+ pass
+
+ def visit_literal_block(self, node: Element) -> None:
+ if node.rawsource != node.astext():
+ # most probably a parsed-literal block -- don't highlight
+ self.in_parsed_literal += 1
+ self.body.append(r'\begin{sphinxalltt}' + CR)
+ else:
+ labels = self.hypertarget_to(node)
+ if isinstance(node.parent, captioned_literal_block):
+ labels += self.hypertarget_to(node.parent)
+ if labels and not self.in_footnote:
+ self.body.append(CR + r'\def\sphinxLiteralBlockLabel{' + labels + '}')
+
+ lang = node.get('language', 'default')
+ linenos = node.get('linenos', False)
+ highlight_args = node.get('highlight_args', {})
+ highlight_args['force'] = node.get('force', False)
+ opts = self.config.highlight_options.get(lang, {})
+
+ hlcode = self.highlighter.highlight_block(
+ node.rawsource, lang, opts=opts, linenos=linenos,
+ location=node, **highlight_args,
+ )
+ if self.in_footnote:
+ self.body.append(CR + r'\sphinxSetupCodeBlockInFootnote')
+ hlcode = hlcode.replace(r'\begin{Verbatim}',
+ r'\begin{sphinxVerbatim}')
+ # if in table raise verbatim flag to avoid "tabulary" environment
+ # and opt for sphinxVerbatimintable to handle caption & long lines
+ elif self.table:
+ self.table.has_problematic = True
+ self.table.has_verbatim = True
+ hlcode = hlcode.replace(r'\begin{Verbatim}',
+ r'\begin{sphinxVerbatimintable}')
+ else:
+ hlcode = hlcode.replace(r'\begin{Verbatim}',
+ r'\begin{sphinxVerbatim}')
+ # get consistent trailer
+ hlcode = hlcode.rstrip()[:-14] # strip \end{Verbatim}
+ if self.table and not self.in_footnote:
+ hlcode += r'\end{sphinxVerbatimintable}'
+ else:
+ hlcode += r'\end{sphinxVerbatim}'
+
+ hllines = str(highlight_args.get('hl_lines', []))[1:-1]
+ if hllines:
+ self.body.append(CR + r'\fvset{hllines={, %s,}}%%' % hllines)
+ self.body.append(CR + hlcode + CR)
+ if hllines:
+ self.body.append(r'\sphinxresetverbatimhllines' + CR)
+ raise nodes.SkipNode
+
+ def depart_literal_block(self, node: Element) -> None:
+ self.body.append(CR + r'\end{sphinxalltt}' + CR)
+ self.in_parsed_literal -= 1
visit_doctest_block = visit_literal_block
depart_doctest_block = depart_literal_block
- def visit_option_argument(self, node: Element) ->None:
+ def visit_line(self, node: Element) -> None:
+ self.body.append(r'\item[] ')
+
+ def depart_line(self, node: Element) -> None:
+ self.body.append(CR)
+
+ def visit_line_block(self, node: Element) -> None:
+ if isinstance(node.parent, nodes.line_block):
+ self.body.append(r'\item[]' + CR)
+ self.body.append(r'\begin{DUlineblock}{\DUlineblockindent}' + CR)
+ else:
+ self.body.append(CR + r'\begin{DUlineblock}{0em}' + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_line_block(self, node: Element) -> None:
+ self.body.append(r'\end{DUlineblock}' + CR)
+
+ def visit_block_quote(self, node: Element) -> None:
+ # If the block quote contains a single object and that object
+ # is a list, then generate a list not a block quote.
+ # This lets us indent lists.
+ done = 0
+ if len(node.children) == 1:
+ child = node.children[0]
+ if isinstance(child, nodes.bullet_list | nodes.enumerated_list):
+ done = 1
+ if not done:
+ self.body.append(r'\begin{quote}' + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_block_quote(self, node: Element) -> None:
+ done = 0
+ if len(node.children) == 1:
+ child = node.children[0]
+ if isinstance(child, nodes.bullet_list | nodes.enumerated_list):
+ done = 1
+ if not done:
+ self.body.append(r'\end{quote}' + CR)
+
+ # option node handling copied from docutils' latex writer
+
+ def visit_option(self, node: Element) -> None:
+ if self.context[-1]:
+ # this is not the first option
+ self.body.append(', ')
+
+ def depart_option(self, node: Element) -> None:
+ # flag that the first option is done.
+ self.context[-1] += 1
+
+ def visit_option_argument(self, node: Element) -> None:
"""The delimiter between an option and its argument."""
+ self.body.append(node.get('delimiter', ' '))
+
+ def depart_option_argument(self, node: Element) -> None:
+ pass
+
+ def visit_option_group(self, node: Element) -> None:
+ self.body.append(r'\item [')
+ # flag for first option
+ self.context.append(0)
+
+ def depart_option_group(self, node: Element) -> None:
+ self.context.pop() # the flag
+ self.body.append('] ')
+
+ def visit_option_list(self, node: Element) -> None:
+ self.body.append(r'\begin{optionlist}{3cm}' + CR)
+ if self.table:
+ self.table.has_problematic = True
+
+ def depart_option_list(self, node: Element) -> None:
+ self.body.append(r'\end{optionlist}' + CR)
+
+ def visit_option_list_item(self, node: Element) -> None:
+ pass
+
+ def depart_option_list_item(self, node: Element) -> None:
+ pass
+
+ def visit_option_string(self, node: Element) -> None:
+ ostring = node.astext()
+ self.body.append(self.encode(ostring))
+ raise nodes.SkipNode
+
+ def visit_description(self, node: Element) -> None:
+ self.body.append(' ')
+
+ def depart_description(self, node: Element) -> None:
+ pass
+
+ def visit_superscript(self, node: Element) -> None:
+ self.body.append(r'$^{\text{')
+
+ def depart_superscript(self, node: Element) -> None:
+ self.body.append('}}$')
+
+ def visit_subscript(self, node: Element) -> None:
+ self.body.append(r'$_{\text{')
+
+ def depart_subscript(self, node: Element) -> None:
+ self.body.append('}}$')
+
+ def visit_inline(self, node: Element) -> None:
+ classes = node.get('classes', []) # type: ignore[var-annotated]
+ if classes == ['menuselection']:
+ self.body.append(r'\sphinxmenuselection{')
+ self.context.append('}')
+ elif classes == ['guilabel']:
+ self.body.append(r'\sphinxguilabel{')
+ self.context.append('}')
+ elif classes == ['accelerator']:
+ self.body.append(r'\sphinxaccelerator{')
+ self.context.append('}')
+ elif classes and not self.in_title:
+ self.body.append(r'\DUrole{' + r'}{\DUrole{'.join(classes) + '}{')
+ self.context.append('}' * len(classes))
+ else:
+ self.context.append('')
+
+ def depart_inline(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+
+ def visit_generated(self, node: Element) -> None:
+ pass
+
+ def depart_generated(self, node: Element) -> None:
+ pass
+
+ def visit_compound(self, node: Element) -> None:
+ pass
+
+ def depart_compound(self, node: Element) -> None:
+ pass
+
+ def visit_container(self, node: Element) -> None:
+ classes = node.get('classes', []) # type: ignore[var-annotated]
+ for c in classes:
+ self.body.append('\n\\begin{sphinxuseclass}{%s}' % c)
+
+ def depart_container(self, node: Element) -> None:
+ classes = node.get('classes', []) # type: ignore[var-annotated]
+ for _c in classes:
+ self.body.append('\n\\end{sphinxuseclass}')
+
+ def visit_decoration(self, node: Element) -> None:
+ pass
+
+ def depart_decoration(self, node: Element) -> None:
+ pass
+
+ # docutils-generated elements that we don't support
+
+ def visit_header(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_footer(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_docinfo(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ # text handling
+
+ def encode(self, text: str) -> str:
+ text = self.escape(text)
+ if self.literal_whitespace:
+ # Insert a blank before the newline, to avoid
+ # ! LaTeX Error: There's no line here to end.
+ text = text.replace(CR, r'~\\' + CR).replace(' ', '~')
+ return text
+
+ def encode_uri(self, text: str) -> str:
+ # TODO: it is probably wrong that this uses texescape.escape()
+ # this must be checked against hyperref package exact dealings
+ # mainly, %, #, {, } and \ need escaping via a \ escape
+ # in \href, the tilde is allowed and must be represented literally
+ return self.encode(text).replace(r'\textasciitilde{}', '~').\
+ replace(r'\sphinxhyphen{}', '-').\
+ replace(r'\textquotesingle{}', "'")
+
+ def visit_Text(self, node: Text) -> None:
+ text = self.encode(node.astext())
+ self.body.append(text)
+
+ def depart_Text(self, node: Text) -> None:
+ pass
+
+ def visit_comment(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_meta(self, node: Element) -> None:
+ # only valid for HTML
+ raise nodes.SkipNode
+
+ def visit_system_message(self, node: Element) -> None:
+ pass
+
+ def depart_system_message(self, node: Element) -> None:
+ self.body.append(CR)
+
+ def visit_math(self, node: Element) -> None:
+ if self.in_title:
+ self.body.append(r'\protect\(%s\protect\)' % node.astext())
+ else:
+ self.body.append(r'\(%s\)' % node.astext())
+ raise nodes.SkipNode
+
+ def visit_math_block(self, node: Element) -> None:
+ if node.get('label'):
+ label = f"equation:{node['docname']}:{node['label']}"
+ else:
+ label = None
+
+ if node.get('nowrap'):
+ if label:
+ self.body.append(r'\label{%s}' % label)
+ self.body.append(node.astext())
+ else:
+ from sphinx.util.math import wrap_displaymath
+ self.body.append(wrap_displaymath(node.astext(), label,
+ self.config.math_number_all))
+ raise nodes.SkipNode
+
+ def visit_math_reference(self, node: Element) -> None:
+ label = f"equation:{node['docname']}:{node['target']}"
+ eqref_format = self.config.math_eqref_format
+ if eqref_format:
+ try:
+ ref = r'\ref{%s}' % label
+ self.body.append(eqref_format.format(number=ref))
+ except KeyError as exc:
+ logger.warning(__('Invalid math_eqref_format: %r'), exc,
+ location=node)
+ self.body.append(r'\eqref{%s}' % label)
+ else:
+ self.body.append(r'\eqref{%s}' % label)
+
+ def depart_math_reference(self, node: Element) -> None:
pass
-from sphinx.builders.latex.nodes import HYPERLINK_SUPPORT_NODES, captioned_literal_block, footnotetext
+# FIXME: Workaround to avoid circular import
+# refs: https://github.com/sphinx-doc/sphinx/issues/5433
+from sphinx.builders.latex.nodes import ( # NoQA: E402 # isort:skip
+ HYPERLINK_SUPPORT_NODES, captioned_literal_block, footnotetext,
+)
diff --git a/sphinx/writers/manpage.py b/sphinx/writers/manpage.py
index 06eeeaee2..2f066e410 100644
--- a/sphinx/writers/manpage.py
+++ b/sphinx/writers/manpage.py
@@ -1,28 +1,42 @@
"""Manual page writer, extended for Sphinx custom nodes."""
+
from __future__ import annotations
+
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes
from docutils.writers.manpage import Translator as BaseTranslator
from docutils.writers.manpage import Writer
+
from sphinx import addnodes
from sphinx.locale import _, admonitionlabels
from sphinx.util import logging
from sphinx.util.docutils import SphinxTranslator
from sphinx.util.i18n import format_date
from sphinx.util.nodes import NodeMatcher
+
if TYPE_CHECKING:
from docutils.nodes import Element
+
from sphinx.builders import Builder
-logger = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
-class ManualPageWriter(Writer):
- def __init__(self, builder: Builder) ->None:
+class ManualPageWriter(Writer): # type: ignore[misc]
+ def __init__(self, builder: Builder) -> None:
super().__init__()
self.builder = builder
+ def translate(self) -> None:
+ transform = NestedInlineTransform(self.document)
+ transform.apply()
+ visitor = self.builder.create_translator(self.document, self.builder)
+ self.visitor = cast(ManualPageTranslator, visitor)
+ self.document.walkabout(visitor)
+ self.output = self.visitor.astext()
+
class NestedInlineTransform:
"""
@@ -36,33 +50,431 @@ class NestedInlineTransform:
<strong>&bar=</strong><emphasis>2</emphasis>
"""
- def __init__(self, document: nodes.document) ->None:
+ def __init__(self, document: nodes.document) -> None:
self.document = document
+ def apply(self, **kwargs: Any) -> None:
+ matcher = NodeMatcher(nodes.literal, nodes.emphasis, nodes.strong)
+ for node in list(matcher.findall(self.document)):
+ if any(matcher(subnode) for subnode in node):
+ pos = node.parent.index(node)
+ for subnode in reversed(list(node)):
+ node.remove(subnode)
+ if matcher(subnode):
+ node.parent.insert(pos + 1, subnode)
+ else:
+ newnode = node.__class__('', '', subnode, **node.attributes)
+ node.parent.insert(pos + 1, newnode)
+ # move node if all children became siblings of the node
+ if not len(node):
+ node.parent.remove(node)
-class ManualPageTranslator(SphinxTranslator, BaseTranslator):
+
+class ManualPageTranslator(SphinxTranslator, BaseTranslator): # type: ignore[misc]
"""
Custom man page translator.
"""
+
_docinfo: dict[str, Any] = {}
- def __init__(self, document: nodes.document, builder: Builder) ->None:
+ def __init__(self, document: nodes.document, builder: Builder) -> None:
super().__init__(document, builder)
+
self.in_productionlist = 0
+
+ # first title is the manpage title
self.section_level = -1
+
+ # docinfo set by man_pages config value
self._docinfo['title'] = self.settings.title
self._docinfo['subtitle'] = self.settings.subtitle
if self.settings.authors:
+ # don't set it if no author given
self._docinfo['author'] = self.settings.authors
self._docinfo['manual_section'] = self.settings.section
+
+ # docinfo set by other config values
self._docinfo['title_upper'] = self._docinfo['title'].upper()
if self.config.today:
self._docinfo['date'] = self.config.today
else:
- self._docinfo['date'] = format_date(self.config.today_fmt or _(
- '%b %d, %Y'), language=self.config.language)
+ self._docinfo['date'] = format_date(self.config.today_fmt or _('%b %d, %Y'),
+ language=self.config.language)
self._docinfo['copyright'] = self.config.copyright
self._docinfo['version'] = self.config.version
self._docinfo['manual_group'] = self.config.project
+
+ # Overwrite admonition label translations with our own
for label, translation in admonitionlabels.items():
self.language.labels[label] = self.deunicode(translation)
+
+ # overwritten -- added quotes around all .TH arguments
+ def header(self) -> str:
+ tmpl = (".TH \"%(title_upper)s\" \"%(manual_section)s\""
+ " \"%(date)s\" \"%(version)s\" \"%(manual_group)s\"\n")
+ if self._docinfo['subtitle']:
+ tmpl += (".SH NAME\n"
+ "%(title)s \\- %(subtitle)s\n")
+ return tmpl % self._docinfo
+
+ def visit_start_of_file(self, node: Element) -> None:
+ pass
+
+ def depart_start_of_file(self, node: Element) -> None:
+ pass
+
+ #############################################################
+ # Domain-specific object descriptions
+ #############################################################
+
+ # Top-level nodes for descriptions
+ ##################################
+
+ def visit_desc(self, node: Element) -> None:
+ self.visit_definition_list(node)
+
+ def depart_desc(self, node: Element) -> None:
+ self.depart_definition_list(node)
+
+ def visit_desc_signature(self, node: Element) -> None:
+ self.visit_definition_list_item(node)
+ self.visit_term(node)
+
+ def depart_desc_signature(self, node: Element) -> None:
+ self.depart_term(node)
+
+ def visit_desc_signature_line(self, node: Element) -> None:
+ pass
+
+ def depart_desc_signature_line(self, node: Element) -> None:
+ self.body.append(' ')
+
+ def visit_desc_content(self, node: Element) -> None:
+ self.visit_definition(node)
+
+ def depart_desc_content(self, node: Element) -> None:
+ self.depart_definition(node)
+
+ def visit_desc_inline(self, node: Element) -> None:
+ pass
+
+ def depart_desc_inline(self, node: Element) -> None:
+ pass
+
+ # Nodes for high-level structure in signatures
+ ##############################################
+
+ def visit_desc_name(self, node: Element) -> None:
+ pass
+
+ def depart_desc_name(self, node: Element) -> None:
+ pass
+
+ def visit_desc_addname(self, node: Element) -> None:
+ pass
+
+ def depart_desc_addname(self, node: Element) -> None:
+ pass
+
+ def visit_desc_type(self, node: Element) -> None:
+ pass
+
+ def depart_desc_type(self, node: Element) -> None:
+ pass
+
+ def visit_desc_returns(self, node: Element) -> None:
+ self.body.append(' -> ')
+
+ def depart_desc_returns(self, node: Element) -> None:
+ pass
+
+ def visit_desc_parameterlist(self, node: Element) -> None:
+ self.body.append('(')
+ self.first_param = 1
+
+ def depart_desc_parameterlist(self, node: Element) -> None:
+ self.body.append(')')
+
+ def visit_desc_type_parameter_list(self, node: Element) -> None:
+ self.body.append('[')
+ self.first_param = 1
+
+ def depart_desc_type_parameter_list(self, node: Element) -> None:
+ self.body.append(']')
+
+ def visit_desc_parameter(self, node: Element) -> None:
+ if not self.first_param:
+ self.body.append(', ')
+ else:
+ self.first_param = 0
+
+ def depart_desc_parameter(self, node: Element) -> None:
+ pass
+
+ def visit_desc_type_parameter(self, node: Element) -> None:
+ self.visit_desc_parameter(node)
+
+ def depart_desc_type_parameter(self, node: Element) -> None:
+ self.depart_desc_parameter(node)
+
+ def visit_desc_optional(self, node: Element) -> None:
+ self.body.append('[')
+
+ def depart_desc_optional(self, node: Element) -> None:
+ self.body.append(']')
+
+ def visit_desc_annotation(self, node: Element) -> None:
+ pass
+
+ def depart_desc_annotation(self, node: Element) -> None:
+ pass
+
+ ##############################################
+
+ def visit_versionmodified(self, node: Element) -> None:
+ self.visit_paragraph(node)
+
+ def depart_versionmodified(self, node: Element) -> None:
+ self.depart_paragraph(node)
+
+ # overwritten -- don't make whole of term bold if it includes strong node
+ def visit_term(self, node: Element) -> None:
+ if any(node.findall(nodes.strong)):
+ self.body.append('\n')
+ else:
+ super().visit_term(node)
+
+ # overwritten -- we don't want source comments to show up
+ def visit_comment(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ # overwritten -- added ensure_eol()
+ def visit_footnote(self, node: Element) -> None:
+ self.ensure_eol()
+ super().visit_footnote(node)
+
+ # overwritten -- handle footnotes rubric
+ def visit_rubric(self, node: Element) -> None:
+ self.ensure_eol()
+ if len(node) == 1 and node.astext() in ('Footnotes', _('Footnotes')):
+ self.body.append('.SH ' + self.deunicode(node.astext()).upper() + '\n')
+ raise nodes.SkipNode
+ self.body.append('.sp\n')
+
+ def depart_rubric(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def visit_seealso(self, node: Element) -> None:
+ self.visit_admonition(node, 'seealso')
+
+ def depart_seealso(self, node: Element) -> None:
+ self.depart_admonition(node)
+
+ def visit_productionlist(self, node: Element) -> None:
+ self.ensure_eol()
+ self.in_productionlist += 1
+ self.body.append('.sp\n.nf\n')
+ productionlist = cast(Iterable[addnodes.production], node)
+ names = (production['tokenname'] for production in productionlist)
+ maxlen = max(len(name) for name in names)
+ lastname = None
+ for production in productionlist:
+ if production['tokenname']:
+ lastname = production['tokenname'].ljust(maxlen)
+ self.body.append(self.defs['strong'][0])
+ self.body.append(self.deunicode(lastname))
+ self.body.append(self.defs['strong'][1])
+ self.body.append(' ::= ')
+ elif lastname is not None:
+ self.body.append('%s ' % (' ' * len(lastname)))
+ production.walkabout(self)
+ self.body.append('\n')
+ self.body.append('\n.fi\n')
+ self.in_productionlist -= 1
+ raise nodes.SkipNode
+
+ def visit_production(self, node: Element) -> None:
+ pass
+
+ def depart_production(self, node: Element) -> None:
+ pass
+
+ # overwritten -- don't emit a warning for images
+ def visit_image(self, node: Element) -> None:
+ if 'alt' in node.attributes:
+ self.body.append(_('[image: %s]') % node['alt'] + '\n')
+ self.body.append(_('[image]') + '\n')
+ raise nodes.SkipNode
+
+ # overwritten -- don't visit inner marked up nodes
+ def visit_reference(self, node: Element) -> None:
+ uri = node.get('refuri', '')
+ is_safe_to_click = uri.startswith(('mailto:', 'http:', 'https:', 'ftp:'))
+ if is_safe_to_click:
+ # OSC 8 link start (using groff's device control directive).
+ self.body.append(fr"\X'tty: link {uri}'")
+
+ self.body.append(self.defs['reference'][0])
+ # avoid repeating escaping code... fine since
+ # visit_Text calls astext() and only works on that afterwards
+ self.visit_Text(node)
+ self.body.append(self.defs['reference'][1])
+
+ if uri and not uri.startswith('#'):
+ # if configured, put the URL after the link
+ if self.config.man_show_urls and node.astext() != uri:
+ if uri.startswith('mailto:'):
+ uri = uri[7:]
+ self.body.extend([
+ ' <',
+ self.defs['strong'][0], uri, self.defs['strong'][1],
+ '>'])
+ if is_safe_to_click:
+ # OSC 8 link end.
+ self.body.append(r"\X'tty: link'")
+ raise nodes.SkipNode
+
+ def visit_number_reference(self, node: Element) -> None:
+ text = nodes.Text(node.get('title', '#'))
+ self.visit_Text(text)
+ raise nodes.SkipNode
+
+ def visit_centered(self, node: Element) -> None:
+ self.ensure_eol()
+ self.body.append('.sp\n.ce\n')
+
+ def depart_centered(self, node: Element) -> None:
+ self.body.append('\n.ce 0\n')
+
+ def visit_compact_paragraph(self, node: Element) -> None:
+ pass
+
+ def depart_compact_paragraph(self, node: Element) -> None:
+ pass
+
+ def visit_download_reference(self, node: Element) -> None:
+ pass
+
+ def depart_download_reference(self, node: Element) -> None:
+ pass
+
+ def visit_toctree(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_index(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_tabular_col_spec(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_glossary(self, node: Element) -> None:
+ pass
+
+ def depart_glossary(self, node: Element) -> None:
+ pass
+
+ def visit_acks(self, node: Element) -> None:
+ bullet_list = cast(nodes.bullet_list, node[0])
+ list_items = cast(Iterable[nodes.list_item], bullet_list)
+ self.ensure_eol()
+ bullet_list = cast(nodes.bullet_list, node[0])
+ list_items = cast(Iterable[nodes.list_item], bullet_list)
+ self.body.append(', '.join(n.astext() for n in list_items) + '.')
+ self.body.append('\n')
+ raise nodes.SkipNode
+
+ def visit_hlist(self, node: Element) -> None:
+ self.visit_bullet_list(node)
+
+ def depart_hlist(self, node: Element) -> None:
+ self.depart_bullet_list(node)
+
+ def visit_hlistcol(self, node: Element) -> None:
+ pass
+
+ def depart_hlistcol(self, node: Element) -> None:
+ pass
+
+ def visit_literal_emphasis(self, node: Element) -> None:
+ return self.visit_emphasis(node)
+
+ def depart_literal_emphasis(self, node: Element) -> None:
+ return self.depart_emphasis(node)
+
+ def visit_literal_strong(self, node: Element) -> None:
+ return self.visit_strong(node)
+
+ def depart_literal_strong(self, node: Element) -> None:
+ return self.depart_strong(node)
+
+ def visit_abbreviation(self, node: Element) -> None:
+ pass
+
+ def depart_abbreviation(self, node: Element) -> None:
+ pass
+
+ def visit_manpage(self, node: Element) -> None:
+ return self.visit_strong(node)
+
+ def depart_manpage(self, node: Element) -> None:
+ return self.depart_strong(node)
+
+ # overwritten: handle section titles better than in 0.6 release
+ def visit_caption(self, node: Element) -> None:
+ if isinstance(node.parent, nodes.container) and node.parent.get('literal_block'):
+ self.body.append('.sp\n')
+ else:
+ super().visit_caption(node)
+
+ def depart_caption(self, node: Element) -> None:
+ if isinstance(node.parent, nodes.container) and node.parent.get('literal_block'):
+ self.body.append('\n')
+ else:
+ super().depart_caption(node)
+
+ # overwritten: handle section titles better than in 0.6 release
+ def visit_title(self, node: Element) -> None:
+ if isinstance(node.parent, addnodes.seealso):
+ self.body.append('.IP "')
+ return None
+ elif isinstance(node.parent, nodes.section):
+ if self.section_level == 0:
+ # skip the document title
+ raise nodes.SkipNode
+ elif self.section_level == 1:
+ self.body.append('.SH %s\n' %
+ self.deunicode(node.astext().upper()))
+ raise nodes.SkipNode
+ return super().visit_title(node)
+
+ def depart_title(self, node: Element) -> None:
+ if isinstance(node.parent, addnodes.seealso):
+ self.body.append('"\n')
+ return None
+ return super().depart_title(node)
+
+ def visit_raw(self, node: Element) -> None:
+ if 'manpage' in node.get('format', '').split():
+ self.body.append(node.astext())
+ raise nodes.SkipNode
+
+ def visit_meta(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_inline(self, node: Element) -> None:
+ pass
+
+ def depart_inline(self, node: Element) -> None:
+ pass
+
+ def visit_math(self, node: Element) -> None:
+ pass
+
+ def depart_math(self, node: Element) -> None:
+ pass
+
+ def visit_math_block(self, node: Element) -> None:
+ self.visit_centered(node)
+
+ def depart_math_block(self, node: Element) -> None:
+ self.depart_centered(node)
diff --git a/sphinx/writers/texinfo.py b/sphinx/writers/texinfo.py
index ff82196df..953115e5d 100644
--- a/sphinx/writers/texinfo.py
+++ b/sphinx/writers/texinfo.py
@@ -1,11 +1,15 @@
"""Custom docutils writer for Texinfo."""
+
from __future__ import annotations
+
import re
import textwrap
from collections.abc import Iterable, Iterator
from os import path
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes, writers
+
from sphinx import __display_version__, addnodes
from sphinx.domains.index import IndexDomain
from sphinx.errors import ExtensionError
@@ -14,12 +18,19 @@ from sphinx.util import logging
from sphinx.util.docutils import SphinxTranslator
from sphinx.util.i18n import format_date
from sphinx.writers.latex import collected_footnote
+
if TYPE_CHECKING:
from docutils.nodes import Element, Node, Text
+
from sphinx.builders.texinfo import TexinfoBuilder
from sphinx.domains import IndexEntry
+
+
logger = logging.getLogger(__name__)
-COPYING = """@quotation
+
+
+COPYING = """\
+@quotation
%(project)s %(release)s, %(date)s
%(author)s
@@ -27,7 +38,9 @@ COPYING = """@quotation
Copyright @copyright{} %(copyright)s
@end quotation
"""
-TEMPLATE = """\\input texinfo @c -*-texinfo-*-
+
+TEMPLATE = """\
+\\input texinfo @c -*-texinfo-*-
@c %%**start of header
@setfilename %(filename)s
@documentencoding UTF-8
@@ -69,59 +82,99 @@ TEMPLATE = """\\input texinfo @c -*-texinfo-*-
"""
-def find_subsections(section: Element) ->list[nodes.section]:
+def find_subsections(section: Element) -> list[nodes.section]:
"""Return a list of subsections for the given ``section``."""
- pass
+ result = []
+ for child in section:
+ if isinstance(child, nodes.section):
+ result.append(child)
+ continue
+ if isinstance(child, nodes.Element):
+ result.extend(find_subsections(child))
+ return result
-def smart_capwords(s: str, sep: (str | None)=None) ->str:
+def smart_capwords(s: str, sep: str | None = None) -> str:
"""Like string.capwords() but does not capitalize words that already
contain a capital letter.
"""
- pass
+ words = s.split(sep)
+ for i, word in enumerate(words):
+ if all(x.islower() for x in word):
+ words[i] = word.capitalize()
+ return (sep or ' ').join(words)
-class TexinfoWriter(writers.Writer):
+class TexinfoWriter(writers.Writer): # type: ignore[misc]
"""Texinfo writer for generating Texinfo documents."""
- supported = 'texinfo', 'texi'
- settings_spec: tuple[str, Any, tuple[tuple[str, list[str], dict[str,
- str]], ...]] = ('Texinfo Specific Options', None, ((
- 'Name of the Info file', ['--texinfo-filename'], {'default': ''}),
- ('Dir entry', ['--texinfo-dir-entry'], {'default': ''}), (
- 'Description', ['--texinfo-dir-description'], {'default': ''}), (
- 'Category', ['--texinfo-dir-category'], {'default': 'Miscellaneous'})))
+
+ supported = ('texinfo', 'texi')
+
+ settings_spec: tuple[str, Any, tuple[tuple[str, list[str], dict[str, str]], ...]] = (
+ 'Texinfo Specific Options', None, (
+ ("Name of the Info file", ['--texinfo-filename'], {'default': ''}),
+ ('Dir entry', ['--texinfo-dir-entry'], {'default': ''}),
+ ('Description', ['--texinfo-dir-description'], {'default': ''}),
+ ('Category', ['--texinfo-dir-category'], {'default':
+ 'Miscellaneous'})))
+
settings_defaults: dict[str, Any] = {}
+
output: str
- visitor_attributes = 'output', 'fragment'
- def __init__(self, builder: TexinfoBuilder) ->None:
+ visitor_attributes = ('output', 'fragment')
+
+ def __init__(self, builder: TexinfoBuilder) -> None:
super().__init__()
self.builder = builder
+ def translate(self) -> None:
+ visitor = self.builder.create_translator(self.document, self.builder)
+ self.visitor = cast(TexinfoTranslator, visitor)
+ self.document.walkabout(visitor)
+ self.visitor.finish()
+ for attr in self.visitor_attributes:
+ setattr(self, attr, getattr(self.visitor, attr))
+
class TexinfoTranslator(SphinxTranslator):
+
ignore_missing_images = False
builder: TexinfoBuilder
- default_elements = {'author': '', 'body': '', 'copying': '', 'date': '',
- 'direntry': '', 'exampleindent': 4, 'filename': '',
- 'paragraphindent': 0, 'preamble': '', 'project': '', 'release': '',
- 'title': ''}
- def __init__(self, document: nodes.document, builder: TexinfoBuilder
- ) ->None:
+ default_elements = {
+ 'author': '',
+ 'body': '',
+ 'copying': '',
+ 'date': '',
+ 'direntry': '',
+ 'exampleindent': 4,
+ 'filename': '',
+ 'paragraphindent': 0,
+ 'preamble': '',
+ 'project': '',
+ 'release': '',
+ 'title': '',
+ }
+
+ def __init__(self, document: nodes.document, builder: TexinfoBuilder) -> None:
super().__init__(document, builder)
self.init_settings()
- self.written_ids: set[str] = set()
+
+ self.written_ids: set[str] = set() # node names and anchors in output
+ # node names and anchors that should be in output
self.referenced_ids: set[str] = set()
- self.indices: list[tuple[str, str]] = []
- self.short_ids: dict[str, str] = {}
- self.node_names: dict[str, str] = {}
- self.node_menus: dict[str, list[str]] = {}
- self.rellinks: dict[str, list[str]] = {}
+ self.indices: list[tuple[str, str]] = [] # (node name, content)
+ self.short_ids: dict[str, str] = {} # anchors --> short ids
+ self.node_names: dict[str, str] = {} # node name --> node's name to display
+ self.node_menus: dict[str, list[str]] = {} # node name --> node's menu entries
+ self.rellinks: dict[str, list[str]] = {} # node name --> (next, previous, up)
+
self.collect_indices()
self.collect_node_names()
self.collect_node_menus()
self.collect_rellinks()
+
self.body: list[str] = []
self.context: list[str] = []
self.descs: list[addnodes.desc] = []
@@ -132,58 +185,931 @@ class TexinfoTranslator(SphinxTranslator):
self.escape_newlines = 0
self.escape_hyphens = 0
self.curfilestack: list[str] = []
- self.footnotestack: list[dict[str, list[collected_footnote | bool]]
- ] = []
+ self.footnotestack: list[dict[str, list[collected_footnote | bool]]] = []
self.in_footnote = 0
self.in_samp = 0
self.handled_abbrs: set[str] = set()
self.colwidths: list[int] = []
- def collect_node_names(self) ->None:
+ def finish(self) -> None:
+ if self.previous_section is None:
+ self.add_menu('Top')
+ for index in self.indices:
+ name, content = index
+ pointers = tuple([name] + self.rellinks[name])
+ self.body.append('\n@node %s,%s,%s,%s\n' % pointers)
+ self.body.append(f'@unnumbered {name}\n\n{content}\n')
+
+ while self.referenced_ids:
+ # handle xrefs with missing anchors
+ r = self.referenced_ids.pop()
+ if r not in self.written_ids:
+ self.body.append('@anchor{{{}}}@w{{{}}}\n'.format(r, ' ' * 30))
+ self.ensure_eol()
+ self.fragment = ''.join(self.body)
+ self.elements['body'] = self.fragment
+ self.output = TEMPLATE % self.elements
+
+ # -- Helper routines
+
+ def init_settings(self) -> None:
+ elements = self.elements = self.default_elements.copy()
+ elements.update({
+ # if empty, the title is set to the first section title
+ 'title': self.settings.title,
+ 'author': self.settings.author,
+ # if empty, use basename of input file
+ 'filename': self.settings.texinfo_filename,
+ 'release': self.escape(self.config.release),
+ 'project': self.escape(self.config.project),
+ 'copyright': self.escape(self.config.copyright),
+ 'date': self.escape(self.config.today or
+ format_date(self.config.today_fmt or _('%b %d, %Y'),
+ language=self.config.language)),
+ })
+ # title
+ title: str = self.settings.title
+ if not title:
+ title_node = self.document.next_node(nodes.title)
+ title = title_node.astext() if title_node else '<untitled>'
+ elements['title'] = self.escape_id(title) or '<untitled>'
+ # filename
+ if not elements['filename']:
+ elements['filename'] = self.document.get('source') or 'untitled'
+ if elements['filename'][-4:] in ('.txt', '.rst'): # type: ignore[index]
+ elements['filename'] = elements['filename'][:-4] # type: ignore[index]
+ elements['filename'] += '.info' # type: ignore[operator]
+ # direntry
+ if self.settings.texinfo_dir_entry:
+ entry = self.format_menu_entry(
+ self.escape_menu(self.settings.texinfo_dir_entry),
+ '(%s)' % elements['filename'],
+ self.escape_arg(self.settings.texinfo_dir_description))
+ elements['direntry'] = ('@dircategory %s\n'
+ '@direntry\n'
+ '%s'
+ '@end direntry\n') % (
+ self.escape_id(self.settings.texinfo_dir_category), entry)
+ elements['copying'] = COPYING % elements
+ # allow the user to override them all
+ elements.update(self.settings.texinfo_elements)
+
+ def collect_node_names(self) -> None:
"""Generates a unique id for each section.
Assigns the attribute ``node_name`` to each section.
"""
- pass
- def collect_node_menus(self) ->None:
+ def add_node_name(name: str) -> str:
+ node_id = self.escape_id(name)
+ nth, suffix = 1, ''
+ while node_id + suffix in self.written_ids or \
+ node_id + suffix in self.node_names:
+ nth += 1
+ suffix = '<%s>' % nth
+ node_id += suffix
+ self.written_ids.add(node_id)
+ self.node_names[node_id] = name
+ return node_id
+
+ # must have a "Top" node
+ self.document['node_name'] = 'Top'
+ add_node_name('Top')
+ add_node_name('top')
+ # each index is a node
+ self.indices = [(add_node_name(name), content)
+ for name, content in self.indices]
+ # each section is also a node
+ for section in self.document.findall(nodes.section):
+ title = cast(nodes.TextElement, section.next_node(nodes.Titular)) # type: ignore[type-var]
+ name = title.astext() if title else '<untitled>'
+ section['node_name'] = add_node_name(name)
+
+ def collect_node_menus(self) -> None:
"""Collect the menu entries for each "node" section."""
- pass
+ node_menus = self.node_menus
+ targets: list[Element] = [self.document]
+ targets.extend(self.document.findall(nodes.section))
+ for node in targets:
+ assert node.get('node_name', False)
+ entries = [s['node_name'] for s in find_subsections(node)]
+ node_menus[node['node_name']] = entries
+ # try to find a suitable "Top" node
+ title = self.document.next_node(nodes.title)
+ top = title.parent if title else self.document
+ if not isinstance(top, nodes.document | nodes.section):
+ top = self.document
+ if top is not self.document:
+ entries = node_menus[top['node_name']]
+ entries += node_menus['Top'][1:]
+ node_menus['Top'] = entries
+ del node_menus[top['node_name']]
+ top['node_name'] = 'Top'
+ # handle the indices
+ for name, _content in self.indices:
+ node_menus[name] = []
+ node_menus['Top'].append(name)
- def collect_rellinks(self) ->None:
+ def collect_rellinks(self) -> None:
"""Collect the relative links (next, previous, up) for each "node"."""
- pass
+ rellinks = self.rellinks
+ node_menus = self.node_menus
+ for id in node_menus:
+ rellinks[id] = ['', '', '']
+ # up's
+ for id, entries in node_menus.items():
+ for e in entries:
+ rellinks[e][2] = id
+ # next's and prev's
+ for id, entries in node_menus.items():
+ for i, id in enumerate(entries):
+ # First child's prev is empty
+ if i != 0:
+ rellinks[id][1] = entries[i - 1]
+ # Last child's next is empty
+ if i != len(entries) - 1:
+ rellinks[id][0] = entries[i + 1]
+ # top's next is its first child
+ try:
+ first = node_menus['Top'][0]
+ except IndexError:
+ pass
+ else:
+ rellinks['Top'][0] = first
+ rellinks[first][1] = 'Top'
- def escape(self, s: str) ->str:
+ # -- Escaping
+ # Which characters to escape depends on the context. In some cases,
+ # namely menus and node names, it's not possible to escape certain
+ # characters.
+
+ def escape(self, s: str) -> str:
"""Return a string with Texinfo command characters escaped."""
- pass
+ s = s.replace('@', '@@')
+ s = s.replace('{', '@{')
+ s = s.replace('}', '@}')
+ # prevent `` and '' quote conversion
+ s = s.replace('``', "`@w{`}")
+ s = s.replace("''", "'@w{'}")
+ return s
- def escape_arg(self, s: str) ->str:
+ def escape_arg(self, s: str) -> str:
"""Return an escaped string suitable for use as an argument
to a Texinfo command.
"""
- pass
+ s = self.escape(s)
+ # commas are the argument delimiters
+ s = s.replace(',', '@comma{}')
+ # normalize white space
+ s = ' '.join(s.split()).strip()
+ return s
- def escape_id(self, s: str) ->str:
+ def escape_id(self, s: str) -> str:
"""Return an escaped string suitable for node names and anchors."""
- pass
+ bad_chars = ',:()'
+ for bc in bad_chars:
+ s = s.replace(bc, ' ')
+ if re.search('[^ .]', s):
+ # remove DOTs if name contains other characters
+ s = s.replace('.', ' ')
+ s = ' '.join(s.split()).strip()
+ return self.escape(s)
- def escape_menu(self, s: str) ->str:
+ def escape_menu(self, s: str) -> str:
"""Return an escaped string suitable for menu entries."""
- pass
+ s = self.escape_arg(s)
+ s = s.replace(':', ';')
+ s = ' '.join(s.split()).strip()
+ return s
- def ensure_eol(self) ->None:
+ def ensure_eol(self) -> None:
"""Ensure the last line in body is terminated by new line."""
- pass
+ if self.body and self.body[-1][-1:] != '\n':
+ self.body.append('\n')
+
+ def format_menu_entry(self, name: str, node_name: str, desc: str) -> str:
+ if name == node_name:
+ s = f'* {name}:: '
+ else:
+ s = f'* {name}: {node_name}. '
+ offset = max((24, (len(name) + 4) % 78))
+ wdesc = '\n'.join(' ' * offset + l for l in
+ textwrap.wrap(desc, width=78 - offset))
+ return s + wdesc.strip() + '\n'
+
+ def add_menu_entries(
+ self,
+ entries: list[str],
+ reg: re.Pattern[str] = re.compile(r'\s+---?\s+'),
+ ) -> None:
+ for entry in entries:
+ name = self.node_names[entry]
+ # special formatting for entries that are divided by an em-dash
+ try:
+ parts = reg.split(name, 1)
+ except TypeError:
+ # could be a gettext proxy
+ parts = [name]
+ if len(parts) == 2:
+ name, desc = parts
+ else:
+ desc = ''
+ name = self.escape_menu(name)
+ desc = self.escape(desc)
+ self.body.append(self.format_menu_entry(name, entry, desc))
+
+ def add_menu(self, node_name: str) -> None:
+ entries = self.node_menus[node_name]
+ if not entries:
+ return
+ self.body.append('\n@menu\n')
+ self.add_menu_entries(entries)
+ if (node_name != 'Top' or
+ not self.node_menus[entries[0]] or
+ self.config.texinfo_no_detailmenu):
+ self.body.append('\n@end menu\n')
+ return
+
+ def _add_detailed_menu(name: str) -> None:
+ entries = self.node_menus[name]
+ if not entries:
+ return
+ self.body.append(f'\n{self.escape(self.node_names[name])}\n\n')
+ self.add_menu_entries(entries)
+ for subentry in entries:
+ _add_detailed_menu(subentry)
+
+ self.body.append('\n@detailmenu\n'
+ ' --- The Detailed Node Listing ---\n')
+ for entry in entries:
+ _add_detailed_menu(entry)
+ self.body.append('\n@end detailmenu\n'
+ '@end menu\n')
+
+ def tex_image_length(self, width_str: str) -> str:
+ match = re.match(r'(\d*\.?\d*)\s*(\S*)', width_str)
+ if not match:
+ # fallback
+ return width_str
+ res = width_str
+ amount, unit = match.groups()[:2]
+ if not unit or unit == "px":
+ # pixels: let TeX alone
+ return ''
+ elif unit == "%":
+ # a4paper: textwidth=418.25368pt
+ res = "%d.0pt" % (float(amount) * 4.1825368)
+ return res
- def get_short_id(self, id: str) ->str:
+ def collect_indices(self) -> None:
+ def generate(content: list[tuple[str, list[IndexEntry]]], collapsed: bool) -> str:
+ ret = ['\n@menu\n']
+ for _letter, entries in content:
+ for entry in entries:
+ if not entry[3]:
+ continue
+ name = self.escape_menu(entry[0])
+ sid = self.get_short_id(f'{entry[2]}:{entry[3]}')
+ desc = self.escape_arg(entry[6])
+ me = self.format_menu_entry(name, sid, desc)
+ ret.append(me)
+ ret.append('@end menu\n')
+ return ''.join(ret)
+
+ if indices_config := self.config.texinfo_domain_indices:
+ if not isinstance(indices_config, bool):
+ check_names = True
+ indices_config = frozenset(indices_config)
+ else:
+ check_names = False
+ for domain_name in sorted(self.builder.env.domains):
+ domain = self.builder.env.domains[domain_name]
+ for index_cls in domain.indices:
+ index_name = f'{domain.name}-{index_cls.name}'
+ if check_names and index_name not in indices_config:
+ continue
+ content, collapsed = index_cls(domain).generate(
+ self.builder.docnames)
+ if content:
+ self.indices.append((
+ index_cls.localname,
+ generate(content, collapsed),
+ ))
+ # only add the main Index if it's not empty
+ domain = cast(IndexDomain, self.builder.env.get_domain('index'))
+ for docname in self.builder.docnames:
+ if domain.entries[docname]:
+ self.indices.append((_('Index'), '\n@printindex ge\n'))
+ break
+
+ # this is copied from the latex writer
+ # TODO: move this to sphinx.util
+
+ def collect_footnotes(
+ self, node: Element,
+ ) -> dict[str, list[collected_footnote | bool]]:
+ def footnotes_under(n: Element) -> Iterator[nodes.footnote]:
+ if isinstance(n, nodes.footnote):
+ yield n
+ else:
+ for c in n.children:
+ if isinstance(c, addnodes.start_of_file):
+ continue
+ elif isinstance(c, nodes.Element):
+ yield from footnotes_under(c)
+ fnotes: dict[str, list[collected_footnote | bool]] = {}
+ for fn in footnotes_under(node):
+ label = cast(nodes.label, fn[0])
+ num = label.astext().strip()
+ fnotes[num] = [collected_footnote('', *fn.children), False]
+ return fnotes
+
+ # -- xref handling
+
+ def get_short_id(self, id: str) -> str:
"""Return a shorter 'id' associated with ``id``."""
+ # Shorter ids improve paragraph filling in places
+ # that the id is hidden by Emacs.
+ try:
+ sid = self.short_ids[id]
+ except KeyError:
+ sid = f'{len(self.short_ids):x}'
+ self.short_ids[id] = sid
+ return sid
+
+ def add_anchor(self, id: str, node: Node) -> None:
+ if id.startswith('index-'):
+ return
+ id = self.curfilestack[-1] + ':' + id
+ eid = self.escape_id(id)
+ sid = self.get_short_id(id)
+ for id in (eid, sid):
+ if id not in self.written_ids:
+ self.body.append('@anchor{%s}' % id)
+ self.written_ids.add(id)
+
+ def add_xref(self, id: str, name: str, node: Node) -> None:
+ name = self.escape_menu(name)
+ sid = self.get_short_id(id)
+ if self.config.texinfo_cross_references:
+ self.body.append(f'@ref{{{sid},,{name}}}')
+ self.referenced_ids.add(sid)
+ self.referenced_ids.add(self.escape_id(id))
+ else:
+ self.body.append(name)
+
+ # -- Visiting
+
+ def visit_document(self, node: Element) -> None:
+ self.footnotestack.append(self.collect_footnotes(node))
+ self.curfilestack.append(node.get('docname', ''))
+ if 'docname' in node:
+ self.add_anchor(':doc', node)
+
+ def depart_document(self, node: Element) -> None:
+ self.footnotestack.pop()
+ self.curfilestack.pop()
+
+ def visit_Text(self, node: Text) -> None:
+ s = self.escape(node.astext())
+ if self.escape_newlines:
+ s = s.replace('\n', ' ')
+ if self.escape_hyphens:
+ # prevent "--" and "---" conversion
+ s = s.replace('-', '@w{-}')
+ self.body.append(s)
+
+ def depart_Text(self, node: Text) -> None:
pass
- headings = ('@unnumbered', '@chapter', '@section', '@subsection',
- '@subsubsection')
- rubrics = '@heading', '@subheading', '@subsubheading'
+
+ def visit_section(self, node: Element) -> None:
+ self.next_section_ids.update(node.get('ids', []))
+ if not self.seen_title:
+ return
+ if self.previous_section:
+ self.add_menu(self.previous_section['node_name'])
+ else:
+ self.add_menu('Top')
+
+ node_name = node['node_name']
+ pointers = tuple([node_name] + self.rellinks[node_name])
+ self.body.append('\n@node %s,%s,%s,%s\n' % pointers)
+ for id in sorted(self.next_section_ids):
+ self.add_anchor(id, node)
+
+ self.next_section_ids.clear()
+ self.previous_section = cast(nodes.section, node)
+ self.section_level += 1
+
+ def depart_section(self, node: Element) -> None:
+ self.section_level -= 1
+
+ headings = (
+ '@unnumbered',
+ '@chapter',
+ '@section',
+ '@subsection',
+ '@subsubsection',
+ )
+
+ rubrics = (
+ '@heading',
+ '@subheading',
+ '@subsubheading',
+ )
+
+ def visit_title(self, node: Element) -> None:
+ if not self.seen_title:
+ self.seen_title = True
+ raise nodes.SkipNode
+ parent = node.parent
+ if isinstance(parent, nodes.table):
+ return
+ if isinstance(parent, nodes.Admonition | nodes.sidebar | nodes.topic):
+ raise nodes.SkipNode
+ if not isinstance(parent, nodes.section):
+ logger.warning(__('encountered title node not in section, topic, table, '
+ 'admonition or sidebar'),
+ location=node)
+ self.visit_rubric(node)
+ else:
+ try:
+ heading = self.headings[self.section_level]
+ except IndexError:
+ heading = self.headings[-1]
+ self.body.append('\n%s ' % heading)
+
+ def depart_title(self, node: Element) -> None:
+ self.body.append('\n\n')
+
+ def visit_rubric(self, node: Element) -> None:
+ if len(node) == 1 and node.astext() in ('Footnotes', _('Footnotes')):
+ raise nodes.SkipNode
+ try:
+ rubric = self.rubrics[self.section_level]
+ except IndexError:
+ rubric = self.rubrics[-1]
+ self.body.append('\n%s ' % rubric)
+ self.escape_newlines += 1
+
+ def depart_rubric(self, node: Element) -> None:
+ self.escape_newlines -= 1
+ self.body.append('\n\n')
+
+ def visit_subtitle(self, node: Element) -> None:
+ self.body.append('\n\n@noindent\n')
+
+ def depart_subtitle(self, node: Element) -> None:
+ self.body.append('\n\n')
+
+ # -- References
+
+ def visit_target(self, node: Element) -> None:
+ # postpone the labels until after the sectioning command
+ parindex = node.parent.index(node)
+ try:
+ try:
+ next = node.parent[parindex + 1]
+ except IndexError:
+ # last node in parent, look at next after parent
+ # (for section of equal level)
+ next = node.parent.parent[node.parent.parent.index(node.parent)]
+ if isinstance(next, nodes.section):
+ if node.get('refid'):
+ self.next_section_ids.add(node['refid'])
+ self.next_section_ids.update(node['ids'])
+ return
+ except (IndexError, AttributeError):
+ pass
+ if 'refuri' in node:
+ return
+ if node.get('refid'):
+ self.add_anchor(node['refid'], node)
+ for id in node['ids']:
+ self.add_anchor(id, node)
+
+ def depart_target(self, node: Element) -> None:
+ pass
+
+ def visit_reference(self, node: Element) -> None:
+ # an xref's target is displayed in Info so we ignore a few
+ # cases for the sake of appearance
+ if isinstance(node.parent, nodes.title | addnodes.desc_type):
+ return
+ if len(node) != 0 and isinstance(node[0], nodes.image):
+ return
+ name = node.get('name', node.astext()).strip()
+ uri = node.get('refuri', '')
+ if not uri and node.get('refid'):
+ uri = '%' + self.curfilestack[-1] + '#' + node['refid']
+ if not uri:
+ return
+ if uri.startswith('mailto:'):
+ uri = self.escape_arg(uri[7:])
+ name = self.escape_arg(name)
+ if not name or name == uri:
+ self.body.append('@email{%s}' % uri)
+ else:
+ self.body.append(f'@email{{{uri},{name}}}')
+ elif uri.startswith('#'):
+ # references to labels in the same document
+ id = self.curfilestack[-1] + ':' + uri[1:]
+ self.add_xref(id, name, node)
+ elif uri.startswith('%'):
+ # references to documents or labels inside documents
+ hashindex = uri.find('#')
+ if hashindex == -1:
+ # reference to the document
+ id = uri[1:] + '::doc'
+ else:
+ # reference to a label
+ id = uri[1:].replace('#', ':')
+ self.add_xref(id, name, node)
+ elif uri.startswith('info:'):
+ # references to an external Info file
+ uri = uri[5:].replace('_', ' ')
+ uri = self.escape_arg(uri)
+ id = 'Top'
+ if '#' in uri:
+ uri, id = uri.split('#', 1)
+ id = self.escape_id(id)
+ name = self.escape_menu(name)
+ if name == id:
+ self.body.append(f'@ref{{{id},,,{uri}}}')
+ else:
+ self.body.append(f'@ref{{{id},,{name},{uri}}}')
+ else:
+ uri = self.escape_arg(uri)
+ name = self.escape_arg(name)
+ show_urls = self.config.texinfo_show_urls
+ if self.in_footnote:
+ show_urls = 'inline'
+ if not name or uri == name:
+ self.body.append('@indicateurl{%s}' % uri)
+ elif show_urls == 'inline':
+ self.body.append(f'@uref{{{uri},{name}}}')
+ elif show_urls == 'no':
+ self.body.append(f'@uref{{{uri},,{name}}}')
+ else:
+ self.body.append(f'{name}@footnote{{{uri}}}')
+ raise nodes.SkipNode
+
+ def depart_reference(self, node: Element) -> None:
+ pass
+
+ def visit_number_reference(self, node: Element) -> None:
+ text = nodes.Text(node.get('title', '#'))
+ self.visit_Text(text)
+ raise nodes.SkipNode
+
+ def visit_title_reference(self, node: Element) -> None:
+ text = node.astext()
+ self.body.append('@cite{%s}' % self.escape_arg(text))
+ raise nodes.SkipNode
+
+ # -- Blocks
+
+ def visit_paragraph(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def depart_paragraph(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def visit_block_quote(self, node: Element) -> None:
+ self.body.append('\n@quotation\n')
+
+ def depart_block_quote(self, node: Element) -> None:
+ self.ensure_eol()
+ self.body.append('@end quotation\n')
+
+ def visit_literal_block(self, node: Element | None) -> None:
+ self.body.append('\n@example\n')
+
+ def depart_literal_block(self, node: Element | None) -> None:
+ self.ensure_eol()
+ self.body.append('@end example\n')
+
visit_doctest_block = visit_literal_block
depart_doctest_block = depart_literal_block
+
+ def visit_line_block(self, node: Element) -> None:
+ if not isinstance(node.parent, nodes.line_block):
+ self.body.append('\n\n')
+ self.body.append('@display\n')
+
+ def depart_line_block(self, node: Element) -> None:
+ self.body.append('@end display\n')
+ if not isinstance(node.parent, nodes.line_block):
+ self.body.append('\n\n')
+
+ def visit_line(self, node: Element) -> None:
+ self.escape_newlines += 1
+
+ def depart_line(self, node: Element) -> None:
+ self.body.append('@w{ }\n')
+ self.escape_newlines -= 1
+
+ # -- Inline
+
+ def visit_strong(self, node: Element) -> None:
+ self.body.append('`')
+
+ def depart_strong(self, node: Element) -> None:
+ self.body.append("'")
+
+ def visit_emphasis(self, node: Element) -> None:
+ if self.in_samp:
+ self.body.append('@var{')
+ self.context.append('}')
+ else:
+ self.body.append('`')
+ self.context.append("'")
+
+ def depart_emphasis(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+
+ def is_samp(self, node: Element) -> bool:
+ return 'samp' in node['classes']
+
+ def visit_literal(self, node: Element) -> None:
+ if self.is_samp(node):
+ self.in_samp += 1
+ self.body.append('@code{')
+
+ def depart_literal(self, node: Element) -> None:
+ if self.is_samp(node):
+ self.in_samp -= 1
+ self.body.append('}')
+
+ def visit_superscript(self, node: Element) -> None:
+ self.body.append('@w{^')
+
+ def depart_superscript(self, node: Element) -> None:
+ self.body.append('}')
+
+ def visit_subscript(self, node: Element) -> None:
+ self.body.append('@w{[')
+
+ def depart_subscript(self, node: Element) -> None:
+ self.body.append(']}')
+
+ # -- Footnotes
+
+ def visit_footnote(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_collected_footnote(self, node: Element) -> None:
+ self.in_footnote += 1
+ self.body.append('@footnote{')
+
+ def depart_collected_footnote(self, node: Element) -> None:
+ self.body.append('}')
+ self.in_footnote -= 1
+
+ def visit_footnote_reference(self, node: Element) -> None:
+ num = node.astext().strip()
+ try:
+ footnode, used = self.footnotestack[-1][num]
+ except (KeyError, IndexError) as exc:
+ raise nodes.SkipNode from exc
+ # footnotes are repeated for each reference
+ footnode.walkabout(self) # type: ignore[union-attr]
+ raise nodes.SkipChildren
+
+ def visit_citation(self, node: Element) -> None:
+ self.body.append('\n')
+ for id in node.get('ids'):
+ self.add_anchor(id, node)
+ self.escape_newlines += 1
+
+ def depart_citation(self, node: Element) -> None:
+ self.escape_newlines -= 1
+
+ def visit_citation_reference(self, node: Element) -> None:
+ self.body.append('@w{[')
+
+ def depart_citation_reference(self, node: Element) -> None:
+ self.body.append(']}')
+
+ # -- Lists
+
+ def visit_bullet_list(self, node: Element) -> None:
+ bullet = node.get('bullet', '*')
+ self.body.append('\n\n@itemize %s\n' % bullet)
+
+ def depart_bullet_list(self, node: Element) -> None:
+ self.ensure_eol()
+ self.body.append('@end itemize\n')
+
+ def visit_enumerated_list(self, node: Element) -> None:
+ # doesn't support Roman numerals
+ enum = node.get('enumtype', 'arabic')
+ starters = {'arabic': '',
+ 'loweralpha': 'a',
+ 'upperalpha': 'A'}
+ start = node.get('start', starters.get(enum, ''))
+ self.body.append('\n\n@enumerate %s\n' % start)
+
+ def depart_enumerated_list(self, node: Element) -> None:
+ self.ensure_eol()
+ self.body.append('@end enumerate\n')
+
+ def visit_list_item(self, node: Element) -> None:
+ self.body.append('\n@item ')
+
+ def depart_list_item(self, node: Element) -> None:
+ pass
+
+ # -- Option List
+
+ def visit_option_list(self, node: Element) -> None:
+ self.body.append('\n\n@table @option\n')
+
+ def depart_option_list(self, node: Element) -> None:
+ self.ensure_eol()
+ self.body.append('@end table\n')
+
+ def visit_option_list_item(self, node: Element) -> None:
+ pass
+
+ def depart_option_list_item(self, node: Element) -> None:
+ pass
+
+ def visit_option_group(self, node: Element) -> None:
+ self.at_item_x = '@item'
+
+ def depart_option_group(self, node: Element) -> None:
+ pass
+
+ def visit_option(self, node: Element) -> None:
+ self.escape_hyphens += 1
+ self.body.append('\n%s ' % self.at_item_x)
+ self.at_item_x = '@itemx'
+
+ def depart_option(self, node: Element) -> None:
+ self.escape_hyphens -= 1
+
+ def visit_option_string(self, node: Element) -> None:
+ pass
+
+ def depart_option_string(self, node: Element) -> None:
+ pass
+
+ def visit_option_argument(self, node: Element) -> None:
+ self.body.append(node.get('delimiter', ' '))
+
+ def depart_option_argument(self, node: Element) -> None:
+ pass
+
+ def visit_description(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def depart_description(self, node: Element) -> None:
+ pass
+
+ # -- Definitions
+
+ def visit_definition_list(self, node: Element) -> None:
+ self.body.append('\n\n@table @asis\n')
+
+ def depart_definition_list(self, node: Element) -> None:
+ self.ensure_eol()
+ self.body.append('@end table\n')
+
+ def visit_definition_list_item(self, node: Element) -> None:
+ self.at_item_x = '@item'
+
+ def depart_definition_list_item(self, node: Element) -> None:
+ pass
+
+ def visit_term(self, node: Element) -> None:
+ for id in node.get('ids'):
+ self.add_anchor(id, node)
+ # anchors and indexes need to go in front
+ for n in node[::]:
+ if isinstance(n, addnodes.index | nodes.target):
+ n.walkabout(self)
+ node.remove(n)
+ self.body.append('\n%s ' % self.at_item_x)
+ self.at_item_x = '@itemx'
+
+ def depart_term(self, node: Element) -> None:
+ pass
+
+ def visit_classifier(self, node: Element) -> None:
+ self.body.append(' : ')
+
+ def depart_classifier(self, node: Element) -> None:
+ pass
+
+ def visit_definition(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def depart_definition(self, node: Element) -> None:
+ pass
+
+ # -- Tables
+
+ def visit_table(self, node: Element) -> None:
+ self.entry_sep = '@item'
+
+ def depart_table(self, node: Element) -> None:
+ self.body.append('\n@end multitable\n\n')
+
+ def visit_tabular_col_spec(self, node: Element) -> None:
+ pass
+
+ def depart_tabular_col_spec(self, node: Element) -> None:
+ pass
+
+ def visit_colspec(self, node: Element) -> None:
+ self.colwidths.append(node['colwidth'])
+ if len(self.colwidths) != self.n_cols:
+ return
+ self.body.append('\n\n@multitable ')
+ for n in self.colwidths:
+ self.body.append('{%s} ' % ('x' * (n + 2)))
+
+ def depart_colspec(self, node: Element) -> None:
+ pass
+
+ def visit_tgroup(self, node: Element) -> None:
+ self.colwidths = []
+ self.n_cols = node['cols']
+
+ def depart_tgroup(self, node: Element) -> None:
+ pass
+
+ def visit_thead(self, node: Element) -> None:
+ self.entry_sep = '@headitem'
+
+ def depart_thead(self, node: Element) -> None:
+ pass
+
+ def visit_tbody(self, node: Element) -> None:
+ pass
+
+ def depart_tbody(self, node: Element) -> None:
+ pass
+
+ def visit_row(self, node: Element) -> None:
+ pass
+
+ def depart_row(self, node: Element) -> None:
+ self.entry_sep = '@item'
+
+ def visit_entry(self, node: Element) -> None:
+ self.body.append('\n%s\n' % self.entry_sep)
+ self.entry_sep = '@tab'
+
+ def depart_entry(self, node: Element) -> None:
+ for _i in range(node.get('morecols', 0)):
+ self.body.append('\n@tab\n')
+
+ # -- Field Lists
+
+ def visit_field_list(self, node: Element) -> None:
+ pass
+
+ def depart_field_list(self, node: Element) -> None:
+ pass
+
+ def visit_field(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def depart_field(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def visit_field_name(self, node: Element) -> None:
+ self.ensure_eol()
+ self.body.append('@*')
+
+ def depart_field_name(self, node: Element) -> None:
+ self.body.append(': ')
+
+ def visit_field_body(self, node: Element) -> None:
+ pass
+
+ def depart_field_body(self, node: Element) -> None:
+ pass
+
+ # -- Admonitions
+
+ def visit_admonition(self, node: Element, name: str = '') -> None:
+ if not name:
+ title = cast(nodes.title, node[0])
+ name = self.escape(title.astext())
+ self.body.append('\n@cartouche\n@quotation %s ' % name)
+
+ def _visit_named_admonition(self, node: Element) -> None:
+ label = admonitionlabels[node.tagname]
+ self.body.append('\n@cartouche\n@quotation %s ' % label)
+
+ def depart_admonition(self, node: Element) -> None:
+ self.ensure_eol()
+ self.body.append('@end quotation\n'
+ '@end cartouche\n')
+
visit_attention = _visit_named_admonition
depart_attention = depart_admonition
visit_caution = _visit_named_admonition
@@ -202,3 +1128,453 @@ class TexinfoTranslator(SphinxTranslator):
depart_tip = depart_admonition
visit_warning = _visit_named_admonition
depart_warning = depart_admonition
+
+ # -- Misc
+
+ def visit_docinfo(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_generated(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_header(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_footer(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_container(self, node: Element) -> None:
+ if node.get('literal_block'):
+ self.body.append('\n\n@float LiteralBlock\n')
+
+ def depart_container(self, node: Element) -> None:
+ if node.get('literal_block'):
+ self.body.append('\n@end float\n\n')
+
+ def visit_decoration(self, node: Element) -> None:
+ pass
+
+ def depart_decoration(self, node: Element) -> None:
+ pass
+
+ def visit_topic(self, node: Element) -> None:
+ # ignore TOC's since we have to have a "menu" anyway
+ if 'contents' in node.get('classes', []):
+ raise nodes.SkipNode
+ title = cast(nodes.title, node[0])
+ self.visit_rubric(title)
+ self.body.append('%s\n' % self.escape(title.astext()))
+ self.depart_rubric(title)
+
+ def depart_topic(self, node: Element) -> None:
+ pass
+
+ def visit_transition(self, node: Element) -> None:
+ self.body.append('\n\n%s\n\n' % ('_' * 66))
+
+ def depart_transition(self, node: Element) -> None:
+ pass
+
+ def visit_attribution(self, node: Element) -> None:
+ self.body.append('\n\n@center --- ')
+
+ def depart_attribution(self, node: Element) -> None:
+ self.body.append('\n\n')
+
+ def visit_raw(self, node: Element) -> None:
+ format = node.get('format', '').split()
+ if 'texinfo' in format or 'texi' in format:
+ self.body.append(node.astext())
+ raise nodes.SkipNode
+
+ def visit_figure(self, node: Element) -> None:
+ self.body.append('\n\n@float Figure\n')
+
+ def depart_figure(self, node: Element) -> None:
+ self.body.append('\n@end float\n\n')
+
+ def visit_caption(self, node: Element) -> None:
+ if (isinstance(node.parent, nodes.figure) or
+ (isinstance(node.parent, nodes.container) and
+ node.parent.get('literal_block'))):
+ self.body.append('\n@caption{')
+ else:
+ logger.warning(__('caption not inside a figure.'),
+ location=node)
+
+ def depart_caption(self, node: Element) -> None:
+ if (isinstance(node.parent, nodes.figure) or
+ (isinstance(node.parent, nodes.container) and
+ node.parent.get('literal_block'))):
+ self.body.append('}\n')
+
+ def visit_image(self, node: Element) -> None:
+ if node['uri'] in self.builder.images:
+ uri = self.builder.images[node['uri']]
+ else:
+ # missing image!
+ if self.ignore_missing_images:
+ return
+ uri = node['uri']
+ if uri.find('://') != -1:
+ # ignore remote images
+ return
+ name, ext = path.splitext(uri)
+ # width and height ignored in non-tex output
+ width = self.tex_image_length(node.get('width', ''))
+ height = self.tex_image_length(node.get('height', ''))
+ alt = self.escape_arg(node.get('alt', ''))
+ filename = f"{self.elements['filename'][:-5]}-figures/{name}" # type: ignore[index]
+ self.body.append('\n@image{%s,%s,%s,%s,%s}\n' %
+ (filename, width, height, alt, ext[1:]))
+
+ def depart_image(self, node: Element) -> None:
+ pass
+
+ def visit_compound(self, node: Element) -> None:
+ pass
+
+ def depart_compound(self, node: Element) -> None:
+ pass
+
+ def visit_sidebar(self, node: Element) -> None:
+ self.visit_topic(node)
+
+ def depart_sidebar(self, node: Element) -> None:
+ self.depart_topic(node)
+
+ def visit_label(self, node: Element) -> None:
+ # label numbering is automatically generated by Texinfo
+ if self.in_footnote:
+ raise nodes.SkipNode
+ self.body.append('@w{(')
+
+ def depart_label(self, node: Element) -> None:
+ self.body.append(')} ')
+
+ def visit_legend(self, node: Element) -> None:
+ pass
+
+ def depart_legend(self, node: Element) -> None:
+ pass
+
+ def visit_substitution_reference(self, node: Element) -> None:
+ pass
+
+ def depart_substitution_reference(self, node: Element) -> None:
+ pass
+
+ def visit_substitution_definition(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_system_message(self, node: Element) -> None:
+ self.body.append('\n@verbatim\n'
+ '<SYSTEM MESSAGE: %s>\n'
+ '@end verbatim\n' % node.astext())
+ raise nodes.SkipNode
+
+ def visit_comment(self, node: Element) -> None:
+ self.body.append('\n')
+ for line in node.astext().splitlines():
+ self.body.append('@c %s\n' % line)
+ raise nodes.SkipNode
+
+ def visit_problematic(self, node: Element) -> None:
+ self.body.append('>>')
+
+ def depart_problematic(self, node: Element) -> None:
+ self.body.append('<<')
+
+ def unimplemented_visit(self, node: Element) -> None:
+ logger.warning(__("unimplemented node type: %r"), node,
+ location=node)
+
+ def unknown_departure(self, node: Node) -> None:
+ pass
+
+ # -- Sphinx specific
+
+ def visit_productionlist(self, node: Element) -> None:
+ self.visit_literal_block(None)
+ productionlist = cast(Iterable[addnodes.production], node)
+ names = (production['tokenname'] for production in productionlist)
+ maxlen = max(len(name) for name in names)
+
+ for production in productionlist:
+ if production['tokenname']:
+ for id in production.get('ids'):
+ self.add_anchor(id, production)
+ s = production['tokenname'].ljust(maxlen) + ' ::='
+ else:
+ s = '%s ' % (' ' * maxlen)
+ self.body.append(self.escape(s))
+ self.body.append(self.escape(production.astext() + '\n'))
+ self.depart_literal_block(None)
+ raise nodes.SkipNode
+
+ def visit_production(self, node: Element) -> None:
+ pass
+
+ def depart_production(self, node: Element) -> None:
+ pass
+
+ def visit_literal_emphasis(self, node: Element) -> None:
+ self.body.append('@code{')
+
+ def depart_literal_emphasis(self, node: Element) -> None:
+ self.body.append('}')
+
+ def visit_literal_strong(self, node: Element) -> None:
+ self.body.append('@code{')
+
+ def depart_literal_strong(self, node: Element) -> None:
+ self.body.append('}')
+
+ def visit_index(self, node: Element) -> None:
+ # terminate the line but don't prevent paragraph breaks
+ if isinstance(node.parent, nodes.paragraph):
+ self.ensure_eol()
+ else:
+ self.body.append('\n')
+ for (_entry_type, value, _target_id, _main, _category_key) in node['entries']:
+ text = self.escape_menu(value)
+ self.body.append('@geindex %s\n' % text)
+
+ def visit_versionmodified(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def depart_versionmodified(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def visit_start_of_file(self, node: Element) -> None:
+ # add a document target
+ self.next_section_ids.add(':doc')
+ self.curfilestack.append(node['docname'])
+ self.footnotestack.append(self.collect_footnotes(node))
+
+ def depart_start_of_file(self, node: Element) -> None:
+ self.curfilestack.pop()
+ self.footnotestack.pop()
+
+ def visit_centered(self, node: Element) -> None:
+ txt = self.escape_arg(node.astext())
+ self.body.append('\n\n@center %s\n\n' % txt)
+ raise nodes.SkipNode
+
+ def visit_seealso(self, node: Element) -> None:
+ self.body.append('\n\n@subsubheading %s\n\n' %
+ admonitionlabels['seealso'])
+
+ def depart_seealso(self, node: Element) -> None:
+ self.body.append('\n')
+
+ def visit_meta(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_glossary(self, node: Element) -> None:
+ pass
+
+ def depart_glossary(self, node: Element) -> None:
+ pass
+
+ def visit_acks(self, node: Element) -> None:
+ bullet_list = cast(nodes.bullet_list, node[0])
+ list_items = cast(Iterable[nodes.list_item], bullet_list)
+ self.body.append('\n\n')
+ self.body.append(', '.join(n.astext() for n in list_items) + '.')
+ self.body.append('\n\n')
+ raise nodes.SkipNode
+
+ #############################################################
+ # Domain-specific object descriptions
+ #############################################################
+
+ # Top-level nodes for descriptions
+ ##################################
+
+ def visit_desc(self, node: addnodes.desc) -> None:
+ self.descs.append(node)
+ self.at_deffnx = '@deffn'
+
+ def depart_desc(self, node: addnodes.desc) -> None:
+ self.descs.pop()
+ self.ensure_eol()
+ self.body.append('@end deffn\n')
+
+ def visit_desc_signature(self, node: Element) -> None:
+ self.escape_hyphens += 1
+ objtype = node.parent['objtype']
+ if objtype != 'describe':
+ for id in node.get('ids'):
+ self.add_anchor(id, node)
+ # use the full name of the objtype for the category
+ try:
+ domain = self.builder.env.get_domain(node.parent['domain'])
+ name = domain.get_type_name(domain.object_types[objtype],
+ self.config.primary_domain == domain.name)
+ except (KeyError, ExtensionError):
+ name = objtype
+ # by convention, the deffn category should be capitalized like a title
+ category = self.escape_arg(smart_capwords(name))
+ self.body.append(f'\n{self.at_deffnx} {{{category}}} ')
+ self.at_deffnx = '@deffnx'
+ self.desc_type_name: str | None = name
+
+ def depart_desc_signature(self, node: Element) -> None:
+ self.body.append("\n")
+ self.escape_hyphens -= 1
+ self.desc_type_name = None
+
+ def visit_desc_signature_line(self, node: Element) -> None:
+ pass
+
+ def depart_desc_signature_line(self, node: Element) -> None:
+ pass
+
+ def visit_desc_content(self, node: Element) -> None:
+ pass
+
+ def depart_desc_content(self, node: Element) -> None:
+ pass
+
+ def visit_desc_inline(self, node: Element) -> None:
+ pass
+
+ def depart_desc_inline(self, node: Element) -> None:
+ pass
+
+ # Nodes for high-level structure in signatures
+ ##############################################
+
+ def visit_desc_name(self, node: Element) -> None:
+ pass
+
+ def depart_desc_name(self, node: Element) -> None:
+ pass
+
+ def visit_desc_addname(self, node: Element) -> None:
+ pass
+
+ def depart_desc_addname(self, node: Element) -> None:
+ pass
+
+ def visit_desc_type(self, node: Element) -> None:
+ pass
+
+ def depart_desc_type(self, node: Element) -> None:
+ pass
+
+ def visit_desc_returns(self, node: Element) -> None:
+ self.body.append(' -> ')
+
+ def depart_desc_returns(self, node: Element) -> None:
+ pass
+
+ def visit_desc_parameterlist(self, node: Element) -> None:
+ self.body.append(' (')
+ self.first_param = 1
+
+ def depart_desc_parameterlist(self, node: Element) -> None:
+ self.body.append(')')
+
+ def visit_desc_type_parameter_list(self, node: Element) -> None:
+ self.body.append(' [')
+ self.first_param = 1
+
+ def depart_desc_type_parameter_list(self, node: Element) -> None:
+ self.body.append(']')
+
+ def visit_desc_parameter(self, node: Element) -> None:
+ if not self.first_param:
+ self.body.append(', ')
+ else:
+ self.first_param = 0
+ text = self.escape(node.astext())
+ # replace no-break spaces with normal ones
+ text = text.replace(' ', '@w{ }')
+ self.body.append(text)
+ raise nodes.SkipNode
+
+ def visit_desc_type_parameter(self, node: Element) -> None:
+ self.visit_desc_parameter(node)
+
+ def visit_desc_optional(self, node: Element) -> None:
+ self.body.append('[')
+
+ def depart_desc_optional(self, node: Element) -> None:
+ self.body.append(']')
+
+ def visit_desc_annotation(self, node: Element) -> None:
+ # Try to avoid duplicating info already displayed by the deffn category.
+ # e.g.
+ # @deffn {Class} Foo
+ # -- instead of --
+ # @deffn {Class} class Foo
+ txt = node.astext().strip()
+ if ((self.descs and txt == self.descs[-1]['objtype']) or
+ (self.desc_type_name and txt in self.desc_type_name.split())):
+ raise nodes.SkipNode
+
+ def depart_desc_annotation(self, node: Element) -> None:
+ pass
+
+ ##############################################
+
+ def visit_inline(self, node: Element) -> None:
+ pass
+
+ def depart_inline(self, node: Element) -> None:
+ pass
+
+ def visit_abbreviation(self, node: Element) -> None:
+ abbr = node.astext()
+ self.body.append('@abbr{')
+ if node.hasattr('explanation') and abbr not in self.handled_abbrs:
+ self.context.append(',%s}' % self.escape_arg(node['explanation']))
+ self.handled_abbrs.add(abbr)
+ else:
+ self.context.append('}')
+
+ def depart_abbreviation(self, node: Element) -> None:
+ self.body.append(self.context.pop())
+
+ def visit_manpage(self, node: Element) -> None:
+ return self.visit_literal_emphasis(node)
+
+ def depart_manpage(self, node: Element) -> None:
+ return self.depart_literal_emphasis(node)
+
+ def visit_download_reference(self, node: Element) -> None:
+ pass
+
+ def depart_download_reference(self, node: Element) -> None:
+ pass
+
+ def visit_hlist(self, node: Element) -> None:
+ self.visit_bullet_list(node)
+
+ def depart_hlist(self, node: Element) -> None:
+ self.depart_bullet_list(node)
+
+ def visit_hlistcol(self, node: Element) -> None:
+ pass
+
+ def depart_hlistcol(self, node: Element) -> None:
+ pass
+
+ def visit_pending_xref(self, node: Element) -> None:
+ pass
+
+ def depart_pending_xref(self, node: Element) -> None:
+ pass
+
+ def visit_math(self, node: Element) -> None:
+ self.body.append('@math{' + self.escape_arg(node.astext()) + '}')
+ raise nodes.SkipNode
+
+ def visit_math_block(self, node: Element) -> None:
+ if node.get('label'):
+ self.add_anchor(node['label'], node)
+ self.body.append('\n\n@example\n%s\n@end example\n\n' %
+ self.escape_arg(node.astext()))
+ raise nodes.SkipNode
diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py
index 2e3317450..67eca45f1 100644
--- a/sphinx/writers/text.py
+++ b/sphinx/writers/text.py
@@ -1,5 +1,6 @@
"""Custom docutils writer for plain text."""
from __future__ import annotations
+
import math
import os
import re
@@ -7,13 +8,17 @@ import textwrap
from collections.abc import Iterable, Iterator, Sequence
from itertools import chain, groupby, pairwise
from typing import TYPE_CHECKING, Any, cast
+
from docutils import nodes, writers
from docutils.utils import column_width
+
from sphinx import addnodes
from sphinx.locale import _, admonitionlabels
from sphinx.util.docutils import SphinxTranslator
+
if TYPE_CHECKING:
from docutils.nodes import Element, Text
+
from sphinx.builders.text import TextBuilder
@@ -22,7 +27,7 @@ class Cell:
It can span multiple columns or multiple lines.
"""
- def __init__(self, text: str='', rowspan: int=1, colspan: int=1) ->None:
+ def __init__(self, text: str = "", rowspan: int = 1, colspan: int = 1) -> None:
self.text = text
self.wrapped: list[str] = []
self.rowspan = rowspan
@@ -30,17 +35,17 @@ class Cell:
self.col: int | None = None
self.row: int | None = None
- def __repr__(self) ->str:
- return (
- f'<Cell {self.text!r} {self.row}v{self.rowspan}/{self.col}>{self.colspan}>'
- )
+ def __repr__(self) -> str:
+ return f"<Cell {self.text!r} {self.row}v{self.rowspan}/{self.col}>{self.colspan}>"
- def __hash__(self) ->int:
+ def __hash__(self) -> int:
return hash((self.col, self.row))
- def __bool__(self) ->bool:
- return (self.text != '' and self.col is not None and self.row is not
- None)
+ def __bool__(self) -> bool:
+ return self.text != '' and self.col is not None and self.row is not None
+
+ def wrap(self, width: int) -> None:
+ self.wrapped = my_wrap(self.text, width)
class Table:
@@ -90,38 +95,42 @@ class Table:
"""
- def __init__(self, colwidth: (list[int] | None)=None) ->None:
+ def __init__(self, colwidth: list[int] | None = None) -> None:
self.lines: list[list[Cell]] = []
self.separator = 0
- self.colwidth: list[int] = colwidth if colwidth is not None else []
+ self.colwidth: list[int] = (colwidth if colwidth is not None else [])
self.current_line = 0
self.current_col = 0
- def add_row(self) ->None:
+ def add_row(self) -> None:
"""Add a row to the table, to use with ``add_cell()``. It is not needed
to call ``add_row()`` before the first ``add_cell()``.
"""
- pass
+ self.current_line += 1
+ self.current_col = 0
- def set_separator(self) ->None:
+ def set_separator(self) -> None:
"""Sets the separator below the current line."""
- pass
+ self.separator = len(self.lines)
- def add_cell(self, cell: Cell) ->None:
+ def add_cell(self, cell: Cell) -> None:
"""Add a cell to the current line, to use with ``add_row()``. To add
a cell spanning multiple lines or rows, simply set the
``cell.colspan`` or ``cell.rowspan`` BEFORE inserting it into
the table.
"""
- pass
+ while self[self.current_line, self.current_col]:
+ self.current_col += 1
+ self[self.current_line, self.current_col] = cell
+ self.current_col += cell.colspan
- def __getitem__(self, pos: tuple[int, int]) ->Cell:
+ def __getitem__(self, pos: tuple[int, int]) -> Cell:
line, col = pos
self._ensure_has_line(line + 1)
self._ensure_has_column(col + 1)
return self.lines[line][col]
- def __setitem__(self, pos: tuple[int, int], cell: Cell) ->None:
+ def __setitem__(self, pos: tuple[int, int], cell: Cell) -> None:
line, col = pos
self._ensure_has_line(line + cell.rowspan)
self._ensure_has_column(col + cell.colspan)
@@ -131,126 +140,245 @@ class Table:
cell.row = line
cell.col = col
- def __repr__(self) ->str:
- return '\n'.join(map(repr, self.lines))
+ def _ensure_has_line(self, line: int) -> None:
+ while len(self.lines) < line:
+ self.lines.append([])
- def cell_width(self, cell: Cell, source: list[int]) ->int:
+ def _ensure_has_column(self, col: int) -> None:
+ for line in self.lines:
+ while len(line) < col:
+ line.append(Cell())
+
+ def __repr__(self) -> str:
+ return "\n".join(map(repr, self.lines))
+
+ def cell_width(self, cell: Cell, source: list[int]) -> int:
"""Give the cell width, according to the given source (either
``self.colwidth`` or ``self.measured_widths``).
This takes into account cells spanning multiple columns.
"""
- pass
+ if cell.row is None or cell.col is None:
+ msg = 'Cell co-ordinates have not been set'
+ raise ValueError(msg)
+ width = 0
+ for i in range(self[cell.row, cell.col].colspan):
+ width += source[cell.col + i]
+ return width + (cell.colspan - 1) * 3
- def rewrap(self) ->None:
+ @property
+ def cells(self) -> Iterator[Cell]:
+ seen: set[Cell] = set()
+ for line in self.lines:
+ for cell in line:
+ if cell and cell not in seen:
+ yield cell
+ seen.add(cell)
+
+ def rewrap(self) -> None:
"""Call ``cell.wrap()`` on all cells, and measure each column width
after wrapping (result written in ``self.measured_widths``).
"""
- pass
+ self.measured_widths = self.colwidth[:]
+ for cell in self.cells:
+ cell.wrap(width=self.cell_width(cell, self.colwidth))
+ if not cell.wrapped:
+ continue
+ if cell.row is None or cell.col is None:
+ msg = 'Cell co-ordinates have not been set'
+ raise ValueError(msg)
+ width = math.ceil(max(column_width(x) for x in cell.wrapped) / cell.colspan)
+ for col in range(cell.col, cell.col + cell.colspan):
+ self.measured_widths[col] = max(self.measured_widths[col], width)
- def physical_lines_for_line(self, line: list[Cell]) ->int:
+ def physical_lines_for_line(self, line: list[Cell]) -> int:
"""For a given line, compute the number of physical lines it spans
due to text wrapping.
"""
- pass
+ physical_lines = 1
+ for cell in line:
+ physical_lines = max(physical_lines, len(cell.wrapped))
+ return physical_lines
- def __str__(self) ->str:
+ def __str__(self) -> str:
out = []
self.rewrap()
- def writesep(char: str='-', lineno: (int | None)=None) ->str:
+ def writesep(char: str = "-", lineno: int | None = None) -> str:
"""Called on the line *before* lineno.
Called with no *lineno* for the last sep.
"""
out: list[str] = []
for colno, width in enumerate(self.measured_widths):
- if lineno is not None and lineno > 0 and self[lineno, colno
- ] is self[lineno - 1, colno]:
- out.append(' ' * (width + 2))
+ if (
+ lineno is not None and
+ lineno > 0 and
+ self[lineno, colno] is self[lineno - 1, colno]
+ ):
+ out.append(" " * (width + 2))
else:
out.append(char * (width + 2))
- head = '+' if out[0][0] == '-' else '|'
- tail = '+' if out[-1][0] == '-' else '|'
- glue = [('+' if left[0] == '-' or right[0] == '-' else '|') for
- left, right in pairwise(out)]
+ head = "+" if out[0][0] == "-" else "|"
+ tail = "+" if out[-1][0] == "-" else "|"
+ glue = [
+ "+" if left[0] == "-" or right[0] == "-" else "|"
+ for left, right in pairwise(out)
+ ]
glue.append(tail)
- return head + ''.join(chain.from_iterable(zip(out, glue, strict
- =False)))
+ return head + "".join(chain.from_iterable(zip(out, glue, strict=False)))
+
for lineno, line in enumerate(self.lines):
if self.separator and lineno == self.separator:
- out.append(writesep('=', lineno))
+ out.append(writesep("=", lineno))
else:
- out.append(writesep('-', lineno))
+ out.append(writesep("-", lineno))
for physical_line in range(self.physical_lines_for_line(line)):
- linestr = ['|']
+ linestr = ["|"]
for colno, cell in enumerate(line):
if cell.col != colno:
continue
- if lineno != cell.row:
- physical_text = ''
+ if lineno != cell.row: # NoQA: SIM114
+ physical_text = ""
elif physical_line >= len(cell.wrapped):
- physical_text = ''
+ physical_text = ""
else:
physical_text = cell.wrapped[physical_line]
- adjust_len = len(physical_text) - column_width(
- physical_text)
- linestr.append(' ' + physical_text.ljust(self.
- cell_width(cell, self.measured_widths) + 1 +
- adjust_len) + '|')
- out.append(''.join(linestr))
- out.append(writesep('-'))
- return '\n'.join(out)
+ adjust_len = len(physical_text) - column_width(physical_text)
+ linestr.append(
+ " " +
+ physical_text.ljust(
+ self.cell_width(cell, self.measured_widths) + 1 + adjust_len,
+ ) + "|",
+ )
+ out.append("".join(linestr))
+ out.append(writesep("-"))
+ return "\n".join(out)
class TextWrapper(textwrap.TextWrapper):
"""Custom subclass that uses a different word separator regex."""
+
wordsep_re = re.compile(
- '(\\s+|(?<=\\s)(?::[a-z-]+:)?`\\S+|[^\\s\\w]*\\w+[a-zA-Z]-(?=\\w+[a-zA-Z])|(?<=[\\w\\!\\"\\\'\\&\\.\\,\\?])-{2,}(?=\\w))'
- )
+ r'(\s+|' # any whitespace
+ r'(?<=\s)(?::[a-z-]+:)?`\S+|' # interpreted text start
+ r'[^\s\w]*\w+[a-zA-Z]-(?=\w+[a-zA-Z])|' # hyphenated words
+ r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash
- def _wrap_chunks(self, chunks: list[str]) ->list[str]:
+ def _wrap_chunks(self, chunks: list[str]) -> list[str]:
"""The original _wrap_chunks uses len() to calculate width.
This method respects wide/fullwidth characters for width adjustment.
"""
- pass
+ lines: list[str] = []
+ if self.width <= 0:
+ raise ValueError("invalid width %r (must be > 0)" % self.width)
- def _break_word(self, word: str, space_left: int) ->tuple[str, str]:
+ chunks.reverse()
+
+ while chunks:
+ cur_line = []
+ cur_len = 0
+
+ if lines:
+ indent = self.subsequent_indent
+ else:
+ indent = self.initial_indent
+
+ width = self.width - column_width(indent)
+
+ if self.drop_whitespace and chunks[-1].strip() == '' and lines:
+ del chunks[-1]
+
+ while chunks:
+ l = column_width(chunks[-1])
+
+ if cur_len + l <= width:
+ cur_line.append(chunks.pop())
+ cur_len += l
+
+ else:
+ break
+
+ if chunks and column_width(chunks[-1]) > width:
+ self._handle_long_word(chunks, cur_line, cur_len, width)
+
+ if self.drop_whitespace and cur_line and cur_line[-1].strip() == '':
+ del cur_line[-1]
+
+ if cur_line:
+ lines.append(indent + ''.join(cur_line))
+
+ return lines
+
+ def _break_word(self, word: str, space_left: int) -> tuple[str, str]:
"""Break line by unicode width instead of len(word)."""
- pass
+ total = 0
+ for i, c in enumerate(word):
+ total += column_width(c)
+ if total > space_left:
+ return word[:i - 1], word[i - 1:]
+ return word, ''
- def _split(self, text: str) ->list[str]:
+ def _split(self, text: str) -> list[str]:
"""Override original method that only split by 'wordsep_re'.
This '_split' splits wide-characters into chunks by one character.
"""
- pass
+ def split(t: str) -> list[str]:
+ return super(TextWrapper, self)._split(t)
+ chunks: list[str] = []
+ for chunk in split(text):
+ for w, g in groupby(chunk, column_width):
+ if w == 1:
+ chunks.extend(split(''.join(g)))
+ else:
+ chunks.extend(list(g))
+ return chunks
- def _handle_long_word(self, reversed_chunks: list[str], cur_line: list[
- str], cur_len: int, width: int) ->None:
+ def _handle_long_word(self, reversed_chunks: list[str], cur_line: list[str],
+ cur_len: int, width: int) -> None:
"""Override original method for using self._break_word() instead of slice."""
- pass
+ space_left = max(width - cur_len, 1)
+ if self.break_long_words:
+ l, r = self._break_word(reversed_chunks[-1], space_left)
+ cur_line.append(l)
+ reversed_chunks[-1] = r
+
+ elif not cur_line:
+ cur_line.append(reversed_chunks.pop())
MAXWIDTH = 70
STDINDENT = 3
-class TextWriter(writers.Writer):
- supported = 'text',
- settings_spec = 'No options here.', '', ()
+def my_wrap(text: str, width: int = MAXWIDTH, **kwargs: Any) -> list[str]:
+ w = TextWrapper(width=width, **kwargs)
+ return w.wrap(text)
+
+
+class TextWriter(writers.Writer): # type: ignore[misc]
+ supported = ('text',)
+ settings_spec = ('No options here.', '', ())
settings_defaults: dict[str, Any] = {}
+
output: str
- def __init__(self, builder: TextBuilder) ->None:
+ def __init__(self, builder: TextBuilder) -> None:
super().__init__()
self.builder = builder
+ def translate(self) -> None:
+ visitor = self.builder.create_translator(self.document, self.builder)
+ self.document.walkabout(visitor)
+ self.output = cast(TextTranslator, visitor).body
+
class TextTranslator(SphinxTranslator):
builder: TextBuilder
- def __init__(self, document: nodes.document, builder: TextBuilder) ->None:
+ def __init__(self, document: nodes.document, builder: TextBuilder) -> None:
super().__init__(document, builder)
+
newlines = self.config.text_newlines
if newlines == 'windows':
self.nl = '\r\n'
@@ -267,23 +395,657 @@ class TextTranslator(SphinxTranslator):
self.sectionlevel = 0
self.lineblocklevel = 0
self.table: Table
+
self.context: list[str] = []
"""Heterogeneous stack.
Used by visit_* and depart_* functions in conjunction with the tree
traversal. Make sure that the pops correspond to the pushes.
"""
+
+ def add_text(self, text: str) -> None:
+ self.states[-1].append((-1, text))
+
+ def new_state(self, indent: int = STDINDENT) -> None:
+ self.states.append([])
+ self.stateindent.append(indent)
+
+ def end_state(
+ self, wrap: bool = True, end: Sequence[str] | None = ('',), first: str | None = None,
+ ) -> None:
+ content = self.states.pop()
+ maxindent = sum(self.stateindent)
+ indent = self.stateindent.pop()
+ result: list[tuple[int, list[str]]] = []
+ toformat: list[str] = []
+
+ def do_format() -> None:
+ if not toformat:
+ return
+ if wrap:
+ res = my_wrap(''.join(toformat), width=MAXWIDTH - maxindent)
+ else:
+ res = ''.join(toformat).splitlines()
+ if end:
+ res += end
+ result.append((indent, res))
+ for itemindent, item in content:
+ if itemindent == -1:
+ toformat.append(item) # type: ignore[arg-type]
+ else:
+ do_format()
+ result.append((indent + itemindent, item)) # type: ignore[arg-type]
+ toformat = []
+ do_format()
+ if first is not None and result:
+ # insert prefix into first line (ex. *, [1], See also, etc.)
+ newindent = result[0][0] - indent
+ if result[0][1] == ['']:
+ result.insert(0, (newindent, [first]))
+ else:
+ text = first + result[0][1].pop(0)
+ result.insert(0, (newindent, [text]))
+
+ self.states[-1].extend(result)
+
+ def visit_document(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_document(self, node: Element) -> None:
+ self.end_state()
+ self.body = self.nl.join(line and (' ' * indent + line)
+ for indent, lines in self.states[0]
+ for line in lines)
+ # XXX header/footer?
+
+ def visit_section(self, node: Element) -> None:
+ self._title_char = self.sectionchars[self.sectionlevel]
+ self.sectionlevel += 1
+
+ def depart_section(self, node: Element) -> None:
+ self.sectionlevel -= 1
+
+ def visit_topic(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_topic(self, node: Element) -> None:
+ self.end_state()
+
visit_sidebar = visit_topic
depart_sidebar = depart_topic
- def _visit_sig_parameter_list(self, node: Element, parameter_group:
- type[Element], sig_open_paren: str, sig_close_paren: str) ->None:
+ def visit_rubric(self, node: Element) -> None:
+ self.new_state(0)
+ self.add_text('-[ ')
+
+ def depart_rubric(self, node: Element) -> None:
+ self.add_text(' ]-')
+ self.end_state()
+
+ def visit_compound(self, node: Element) -> None:
+ pass
+
+ def depart_compound(self, node: Element) -> None:
+ pass
+
+ def visit_glossary(self, node: Element) -> None:
+ pass
+
+ def depart_glossary(self, node: Element) -> None:
+ pass
+
+ def visit_title(self, node: Element) -> None:
+ if isinstance(node.parent, nodes.Admonition):
+ self.add_text(node.astext() + ': ')
+ raise nodes.SkipNode
+ self.new_state(0)
+
+ def get_section_number_string(self, node: Element) -> str:
+ if isinstance(node.parent, nodes.section):
+ anchorname = '#' + node.parent['ids'][0]
+ numbers = self.builder.secnumbers.get(anchorname)
+ if numbers is None:
+ numbers = self.builder.secnumbers.get('')
+ if numbers is not None:
+ return '.'.join(map(str, numbers)) + self.secnumber_suffix
+ return ''
+
+ def depart_title(self, node: Element) -> None:
+ if isinstance(node.parent, nodes.section):
+ char = self._title_char
+ else:
+ char = '^'
+ text = ''
+ text = ''.join(x[1] for x in self.states.pop() if x[0] == -1) # type: ignore[misc]
+ if self.add_secnumbers:
+ text = self.get_section_number_string(node) + text
+ self.stateindent.pop()
+ title = ['', text, '%s' % (char * column_width(text)), '']
+ if len(self.states) == 2 and len(self.states[-1]) == 0:
+ # remove an empty line before title if it is first section title in the document
+ title.pop(0)
+ self.states[-1].append((0, title))
+
+ def visit_subtitle(self, node: Element) -> None:
+ pass
+
+ def depart_subtitle(self, node: Element) -> None:
+ pass
+
+ def visit_attribution(self, node: Element) -> None:
+ self.add_text('-- ')
+
+ def depart_attribution(self, node: Element) -> None:
+ pass
+
+ #############################################################
+ # Domain-specific object descriptions
+ #############################################################
+
+ # Top-level nodes
+ #################
+
+ def visit_desc(self, node: Element) -> None:
+ pass
+
+ def depart_desc(self, node: Element) -> None:
+ pass
+
+ def visit_desc_signature(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_desc_signature(self, node: Element) -> None:
+ # XXX: wrap signatures in a way that makes sense
+ self.end_state(wrap=False, end=None)
+
+ def visit_desc_signature_line(self, node: Element) -> None:
+ pass
+
+ def depart_desc_signature_line(self, node: Element) -> None:
+ self.add_text('\n')
+
+ def visit_desc_content(self, node: Element) -> None:
+ self.new_state()
+ self.add_text(self.nl)
+
+ def depart_desc_content(self, node: Element) -> None:
+ self.end_state()
+
+ def visit_desc_inline(self, node: Element) -> None:
+ pass
+
+ def depart_desc_inline(self, node: Element) -> None:
+ pass
+
+ # Nodes for high-level structure in signatures
+ ##############################################
+
+ def visit_desc_name(self, node: Element) -> None:
+ pass
+
+ def depart_desc_name(self, node: Element) -> None:
+ pass
+
+ def visit_desc_addname(self, node: Element) -> None:
+ pass
+
+ def depart_desc_addname(self, node: Element) -> None:
+ pass
+
+ def visit_desc_type(self, node: Element) -> None:
+ pass
+
+ def depart_desc_type(self, node: Element) -> None:
+ pass
+
+ def visit_desc_returns(self, node: Element) -> None:
+ self.add_text(' -> ')
+
+ def depart_desc_returns(self, node: Element) -> None:
+ pass
+
+ def _visit_sig_parameter_list(
+ self,
+ node: Element,
+ parameter_group: type[Element],
+ sig_open_paren: str,
+ sig_close_paren: str,
+ ) -> None:
"""Visit a signature parameters or type parameters list.
The *parameter_group* value is the type of a child node acting as a required parameter
or as a set of contiguous optional parameters.
"""
+ self.add_text(sig_open_paren)
+ self.is_first_param = True
+ self.optional_param_level = 0
+ self.params_left_at_level = 0
+ self.param_group_index = 0
+ # Counts as what we call a parameter group are either a required parameter, or a
+ # set of contiguous optional ones.
+ self.list_is_required_param = [isinstance(c, parameter_group) for c in node.children]
+ self.required_params_left = sum(self.list_is_required_param)
+ self.param_separator = ', '
+ self.multi_line_parameter_list = node.get('multi_line_parameter_list', False)
+ if self.multi_line_parameter_list:
+ self.param_separator = self.param_separator.rstrip()
+ self.context.append(sig_close_paren)
+
+ def _depart_sig_parameter_list(self, node: Element) -> None:
+ sig_close_paren = self.context.pop()
+ self.add_text(sig_close_paren)
+
+ def visit_desc_parameterlist(self, node: Element) -> None:
+ self._visit_sig_parameter_list(node, addnodes.desc_parameter, '(', ')')
+
+ def depart_desc_parameterlist(self, node: Element) -> None:
+ self._depart_sig_parameter_list(node)
+
+ def visit_desc_type_parameter_list(self, node: Element) -> None:
+ self._visit_sig_parameter_list(node, addnodes.desc_type_parameter, '[', ']')
+
+ def depart_desc_type_parameter_list(self, node: Element) -> None:
+ self._depart_sig_parameter_list(node)
+
+ def visit_desc_parameter(self, node: Element) -> None:
+ on_separate_line = self.multi_line_parameter_list
+ if on_separate_line and not (self.is_first_param and self.optional_param_level > 0):
+ self.new_state()
+ if self.is_first_param:
+ self.is_first_param = False
+ elif not on_separate_line and not self.required_params_left:
+ self.add_text(self.param_separator)
+ if self.optional_param_level == 0:
+ self.required_params_left -= 1
+ else:
+ self.params_left_at_level -= 1
+
+ self.add_text(node.astext())
+
+ is_required = self.list_is_required_param[self.param_group_index]
+ if on_separate_line:
+ is_last_group = self.param_group_index + 1 == len(self.list_is_required_param)
+ next_is_required = (
+ not is_last_group
+ and self.list_is_required_param[self.param_group_index + 1]
+ )
+ opt_param_left_at_level = self.params_left_at_level > 0
+ if opt_param_left_at_level or is_required and (is_last_group or next_is_required):
+ self.add_text(self.param_separator)
+ self.end_state(wrap=False, end=None)
+
+ elif self.required_params_left:
+ self.add_text(self.param_separator)
+
+ if is_required:
+ self.param_group_index += 1
+ raise nodes.SkipNode
+
+ def visit_desc_type_parameter(self, node: Element) -> None:
+ self.visit_desc_parameter(node)
+
+ def visit_desc_optional(self, node: Element) -> None:
+ self.params_left_at_level = sum(isinstance(c, addnodes.desc_parameter)
+ for c in node.children)
+ self.optional_param_level += 1
+ self.max_optional_param_level = self.optional_param_level
+ if self.multi_line_parameter_list:
+ # If the first parameter is optional, start a new line and open the bracket.
+ if self.is_first_param:
+ self.new_state()
+ self.add_text('[')
+ # Else, if there remains at least one required parameter, append the
+ # parameter separator, open a new bracket, and end the line.
+ elif self.required_params_left:
+ self.add_text(self.param_separator)
+ self.add_text('[')
+ self.end_state(wrap=False, end=None)
+ # Else, open a new bracket, append the parameter separator, and end the
+ # line.
+ else:
+ self.add_text('[')
+ self.add_text(self.param_separator)
+ self.end_state(wrap=False, end=None)
+ else:
+ self.add_text('[')
+
+ def depart_desc_optional(self, node: Element) -> None:
+ self.optional_param_level -= 1
+ if self.multi_line_parameter_list:
+ # If it's the first time we go down one level, add the separator before the
+ # bracket.
+ if self.optional_param_level == self.max_optional_param_level - 1:
+ self.add_text(self.param_separator)
+ self.add_text(']')
+ # End the line if we have just closed the last bracket of this group of
+ # optional parameters.
+ if self.optional_param_level == 0:
+ self.end_state(wrap=False, end=None)
+
+ else:
+ self.add_text(']')
+ if self.optional_param_level == 0:
+ self.param_group_index += 1
+
+ def visit_desc_annotation(self, node: Element) -> None:
+ pass
+
+ def depart_desc_annotation(self, node: Element) -> None:
+ pass
+
+ ##############################################
+
+ def visit_figure(self, node: Element) -> None:
+ self.new_state()
+
+ def depart_figure(self, node: Element) -> None:
+ self.end_state()
+
+ def visit_caption(self, node: Element) -> None:
+ pass
+
+ def depart_caption(self, node: Element) -> None:
+ pass
+
+ def visit_productionlist(self, node: Element) -> None:
+ self.new_state()
+ productionlist = cast(Iterable[addnodes.production], node)
+ names = (production['tokenname'] for production in productionlist)
+ maxlen = max(len(name) for name in names)
+ lastname = None
+ for production in productionlist:
+ if production['tokenname']:
+ self.add_text(production['tokenname'].ljust(maxlen) + ' ::=')
+ lastname = production['tokenname']
+ elif lastname is not None:
+ self.add_text('%s ' % (' ' * len(lastname)))
+ self.add_text(production.astext() + self.nl)
+ self.end_state(wrap=False)
+ raise nodes.SkipNode
+
+ def visit_footnote(self, node: Element) -> None:
+ label = cast(nodes.label, node[0])
+ self._footnote = label.astext().strip()
+ self.new_state(len(self._footnote) + 3)
+
+ def depart_footnote(self, node: Element) -> None:
+ self.end_state(first='[%s] ' % self._footnote)
+
+ def visit_citation(self, node: Element) -> None:
+ if len(node) and isinstance(node[0], nodes.label):
+ self._citlabel = node[0].astext()
+ else:
+ self._citlabel = ''
+ self.new_state(len(self._citlabel) + 3)
+
+ def depart_citation(self, node: Element) -> None:
+ self.end_state(first='[%s] ' % self._citlabel)
+
+ def visit_label(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_legend(self, node: Element) -> None:
+ pass
+
+ def depart_legend(self, node: Element) -> None:
+ pass
+
+ # XXX: option list could use some better styling
+
+ def visit_option_list(self, node: Element) -> None:
+ pass
+
+ def depart_option_list(self, node: Element) -> None:
+ pass
+
+ def visit_option_list_item(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_option_list_item(self, node: Element) -> None:
+ self.end_state()
+
+ def visit_option_group(self, node: Element) -> None:
+ self._firstoption = True
+
+ def depart_option_group(self, node: Element) -> None:
+ self.add_text(' ')
+
+ def visit_option(self, node: Element) -> None:
+ if self._firstoption:
+ self._firstoption = False
+ else:
+ self.add_text(', ')
+
+ def depart_option(self, node: Element) -> None:
pass
+
+ def visit_option_string(self, node: Element) -> None:
+ pass
+
+ def depart_option_string(self, node: Element) -> None:
+ pass
+
+ def visit_option_argument(self, node: Element) -> None:
+ self.add_text(node['delimiter'])
+
+ def depart_option_argument(self, node: Element) -> None:
+ pass
+
+ def visit_description(self, node: Element) -> None:
+ pass
+
+ def depart_description(self, node: Element) -> None:
+ pass
+
+ def visit_tabular_col_spec(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_colspec(self, node: Element) -> None:
+ self.table.colwidth.append(node["colwidth"])
+ raise nodes.SkipNode
+
+ def visit_tgroup(self, node: Element) -> None:
+ pass
+
+ def depart_tgroup(self, node: Element) -> None:
+ pass
+
+ def visit_thead(self, node: Element) -> None:
+ pass
+
+ def depart_thead(self, node: Element) -> None:
+ pass
+
+ def visit_tbody(self, node: Element) -> None:
+ self.table.set_separator()
+
+ def depart_tbody(self, node: Element) -> None:
+ pass
+
+ def visit_row(self, node: Element) -> None:
+ if self.table.lines:
+ self.table.add_row()
+
+ def depart_row(self, node: Element) -> None:
+ pass
+
+ def visit_entry(self, node: Element) -> None:
+ self.entry = Cell(
+ rowspan=node.get("morerows", 0) + 1, colspan=node.get("morecols", 0) + 1,
+ )
+ self.new_state(0)
+
+ def depart_entry(self, node: Element) -> None:
+ text = self.nl.join(self.nl.join(x[1]) for x in self.states.pop())
+ self.stateindent.pop()
+ self.entry.text = text
+ self.table.add_cell(self.entry)
+ del self.entry
+
+ def visit_table(self, node: Element) -> None:
+ if hasattr(self, 'table'):
+ msg = 'Nested tables are not supported.'
+ raise NotImplementedError(msg)
+ self.new_state(0)
+ self.table = Table()
+
+ def depart_table(self, node: Element) -> None:
+ self.add_text(str(self.table))
+ del self.table
+ self.end_state(wrap=False)
+
+ def visit_acks(self, node: Element) -> None:
+ bullet_list = cast(nodes.bullet_list, node[0])
+ list_items = cast(Iterable[nodes.list_item], bullet_list)
+ self.new_state(0)
+ self.add_text(', '.join(n.astext() for n in list_items) + '.')
+ self.end_state()
+ raise nodes.SkipNode
+
+ def visit_image(self, node: Element) -> None:
+ if 'alt' in node.attributes:
+ self.add_text(_('[image: %s]') % node['alt'])
+ self.add_text(_('[image]'))
+ raise nodes.SkipNode
+
+ def visit_transition(self, node: Element) -> None:
+ indent = sum(self.stateindent)
+ self.new_state(0)
+ self.add_text('=' * (MAXWIDTH - indent))
+ self.end_state()
+ raise nodes.SkipNode
+
+ def visit_bullet_list(self, node: Element) -> None:
+ self.list_counter.append(-1)
+
+ def depart_bullet_list(self, node: Element) -> None:
+ self.list_counter.pop()
+
+ def visit_enumerated_list(self, node: Element) -> None:
+ self.list_counter.append(node.get('start', 1) - 1)
+
+ def depart_enumerated_list(self, node: Element) -> None:
+ self.list_counter.pop()
+
+ def visit_definition_list(self, node: Element) -> None:
+ self.list_counter.append(-2)
+
+ def depart_definition_list(self, node: Element) -> None:
+ self.list_counter.pop()
+
+ def visit_list_item(self, node: Element) -> None:
+ if self.list_counter[-1] == -1:
+ # bullet list
+ self.new_state(2)
+ elif self.list_counter[-1] == -2:
+ # definition list
+ pass
+ else:
+ # enumerated list
+ self.list_counter[-1] += 1
+ self.new_state(len(str(self.list_counter[-1])) + 2)
+
+ def depart_list_item(self, node: Element) -> None:
+ if self.list_counter[-1] == -1:
+ self.end_state(first='* ')
+ elif self.list_counter[-1] == -2:
+ pass
+ else:
+ self.end_state(first='%s. ' % self.list_counter[-1])
+
+ def visit_definition_list_item(self, node: Element) -> None:
+ self._classifier_count_in_li = len(list(node.findall(nodes.classifier)))
+
+ def depart_definition_list_item(self, node: Element) -> None:
+ pass
+
+ def visit_term(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_term(self, node: Element) -> None:
+ if not self._classifier_count_in_li:
+ self.end_state(end=None)
+
+ def visit_classifier(self, node: Element) -> None:
+ self.add_text(' : ')
+
+ def depart_classifier(self, node: Element) -> None:
+ self._classifier_count_in_li -= 1
+ if not self._classifier_count_in_li:
+ self.end_state(end=None)
+
+ def visit_definition(self, node: Element) -> None:
+ self.new_state()
+
+ def depart_definition(self, node: Element) -> None:
+ self.end_state()
+
+ def visit_field_list(self, node: Element) -> None:
+ pass
+
+ def depart_field_list(self, node: Element) -> None:
+ pass
+
+ def visit_field(self, node: Element) -> None:
+ pass
+
+ def depart_field(self, node: Element) -> None:
+ pass
+
+ def visit_field_name(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_field_name(self, node: Element) -> None:
+ self.add_text(':')
+ self.end_state(end=None)
+
+ def visit_field_body(self, node: Element) -> None:
+ self.new_state()
+
+ def depart_field_body(self, node: Element) -> None:
+ self.end_state()
+
+ def visit_centered(self, node: Element) -> None:
+ pass
+
+ def depart_centered(self, node: Element) -> None:
+ pass
+
+ def visit_hlist(self, node: Element) -> None:
+ pass
+
+ def depart_hlist(self, node: Element) -> None:
+ pass
+
+ def visit_hlistcol(self, node: Element) -> None:
+ pass
+
+ def depart_hlistcol(self, node: Element) -> None:
+ pass
+
+ def visit_admonition(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_admonition(self, node: Element) -> None:
+ self.end_state()
+
+ def _visit_admonition(self, node: Element) -> None:
+ self.new_state(2)
+
+ def _depart_admonition(self, node: Element) -> None:
+ label = admonitionlabels[node.tagname]
+ indent = sum(self.stateindent) + len(label)
+ if (len(self.states[-1]) == 1 and
+ self.states[-1][0][0] == 0 and
+ MAXWIDTH - indent >= sum(len(s) for s in self.states[-1][0][1])):
+ # short text: append text after admonition label
+ self.stateindent[-1] += len(label)
+ self.end_state(first=label + ': ')
+ else:
+ # long text: append label before the block
+ self.states[-1].insert(0, (0, [self.nl]))
+ self.end_state(first=label + ':')
+
visit_attention = _visit_admonition
depart_attention = _depart_admonition
visit_caution = _visit_admonition
@@ -304,3 +1066,230 @@ class TextTranslator(SphinxTranslator):
depart_warning = _depart_admonition
visit_seealso = _visit_admonition
depart_seealso = _depart_admonition
+
+ def visit_versionmodified(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_versionmodified(self, node: Element) -> None:
+ self.end_state()
+
+ def visit_literal_block(self, node: Element) -> None:
+ self.new_state()
+
+ def depart_literal_block(self, node: Element) -> None:
+ self.end_state(wrap=False)
+
+ def visit_doctest_block(self, node: Element) -> None:
+ self.new_state(0)
+
+ def depart_doctest_block(self, node: Element) -> None:
+ self.end_state(wrap=False)
+
+ def visit_line_block(self, node: Element) -> None:
+ self.new_state()
+ self.lineblocklevel += 1
+
+ def depart_line_block(self, node: Element) -> None:
+ self.lineblocklevel -= 1
+ self.end_state(wrap=False, end=None)
+ if not self.lineblocklevel:
+ self.add_text('\n')
+
+ def visit_line(self, node: Element) -> None:
+ pass
+
+ def depart_line(self, node: Element) -> None:
+ self.add_text('\n')
+
+ def visit_block_quote(self, node: Element) -> None:
+ self.new_state()
+
+ def depart_block_quote(self, node: Element) -> None:
+ self.end_state()
+
+ def visit_compact_paragraph(self, node: Element) -> None:
+ pass
+
+ def depart_compact_paragraph(self, node: Element) -> None:
+ pass
+
+ def visit_paragraph(self, node: Element) -> None:
+ if not isinstance(node.parent, nodes.Admonition) or \
+ isinstance(node.parent, addnodes.seealso):
+ self.new_state(0)
+
+ def depart_paragraph(self, node: Element) -> None:
+ if not isinstance(node.parent, nodes.Admonition) or \
+ isinstance(node.parent, addnodes.seealso):
+ self.end_state()
+
+ def visit_target(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_index(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_toctree(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_substitution_definition(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_pending_xref(self, node: Element) -> None:
+ pass
+
+ def depart_pending_xref(self, node: Element) -> None:
+ pass
+
+ def visit_reference(self, node: Element) -> None:
+ if self.add_secnumbers:
+ numbers = node.get("secnumber")
+ if numbers is not None:
+ self.add_text('.'.join(map(str, numbers)) + self.secnumber_suffix)
+
+ def depart_reference(self, node: Element) -> None:
+ pass
+
+ def visit_number_reference(self, node: Element) -> None:
+ text = nodes.Text(node.get('title', '#'))
+ self.visit_Text(text)
+ raise nodes.SkipNode
+
+ def visit_download_reference(self, node: Element) -> None:
+ pass
+
+ def depart_download_reference(self, node: Element) -> None:
+ pass
+
+ def visit_emphasis(self, node: Element) -> None:
+ self.add_text('*')
+
+ def depart_emphasis(self, node: Element) -> None:
+ self.add_text('*')
+
+ def visit_literal_emphasis(self, node: Element) -> None:
+ self.add_text('*')
+
+ def depart_literal_emphasis(self, node: Element) -> None:
+ self.add_text('*')
+
+ def visit_strong(self, node: Element) -> None:
+ self.add_text('**')
+
+ def depart_strong(self, node: Element) -> None:
+ self.add_text('**')
+
+ def visit_literal_strong(self, node: Element) -> None:
+ self.add_text('**')
+
+ def depart_literal_strong(self, node: Element) -> None:
+ self.add_text('**')
+
+ def visit_abbreviation(self, node: Element) -> None:
+ self.add_text('')
+
+ def depart_abbreviation(self, node: Element) -> None:
+ if node.hasattr('explanation'):
+ self.add_text(' (%s)' % node['explanation'])
+
+ def visit_manpage(self, node: Element) -> None:
+ return self.visit_literal_emphasis(node)
+
+ def depart_manpage(self, node: Element) -> None:
+ return self.depart_literal_emphasis(node)
+
+ def visit_title_reference(self, node: Element) -> None:
+ self.add_text('*')
+
+ def depart_title_reference(self, node: Element) -> None:
+ self.add_text('*')
+
+ def visit_literal(self, node: Element) -> None:
+ self.add_text('"')
+
+ def depart_literal(self, node: Element) -> None:
+ self.add_text('"')
+
+ def visit_subscript(self, node: Element) -> None:
+ self.add_text('_')
+
+ def depart_subscript(self, node: Element) -> None:
+ pass
+
+ def visit_superscript(self, node: Element) -> None:
+ self.add_text('^')
+
+ def depart_superscript(self, node: Element) -> None:
+ pass
+
+ def visit_footnote_reference(self, node: Element) -> None:
+ self.add_text('[%s]' % node.astext())
+ raise nodes.SkipNode
+
+ def visit_citation_reference(self, node: Element) -> None:
+ self.add_text('[%s]' % node.astext())
+ raise nodes.SkipNode
+
+ def visit_Text(self, node: Text) -> None:
+ self.add_text(node.astext())
+
+ def depart_Text(self, node: Text) -> None:
+ pass
+
+ def visit_generated(self, node: Element) -> None:
+ pass
+
+ def depart_generated(self, node: Element) -> None:
+ pass
+
+ def visit_inline(self, node: Element) -> None:
+ if 'xref' in node['classes'] or 'term' in node['classes']:
+ self.add_text('*')
+
+ def depart_inline(self, node: Element) -> None:
+ if 'xref' in node['classes'] or 'term' in node['classes']:
+ self.add_text('*')
+
+ def visit_container(self, node: Element) -> None:
+ pass
+
+ def depart_container(self, node: Element) -> None:
+ pass
+
+ def visit_problematic(self, node: Element) -> None:
+ self.add_text('>>')
+
+ def depart_problematic(self, node: Element) -> None:
+ self.add_text('<<')
+
+ def visit_system_message(self, node: Element) -> None:
+ self.new_state(0)
+ self.add_text('<SYSTEM MESSAGE: %s>' % node.astext())
+ self.end_state()
+ raise nodes.SkipNode
+
+ def visit_comment(self, node: Element) -> None:
+ raise nodes.SkipNode
+
+ def visit_meta(self, node: Element) -> None:
+ # only valid for HTML
+ raise nodes.SkipNode
+
+ def visit_raw(self, node: Element) -> None:
+ if 'text' in node.get('format', '').split():
+ self.new_state(0)
+ self.add_text(node.astext())
+ self.end_state(wrap=False)
+ raise nodes.SkipNode
+
+ def visit_math(self, node: Element) -> None:
+ pass
+
+ def depart_math(self, node: Element) -> None:
+ pass
+
+ def visit_math_block(self, node: Element) -> None:
+ self.new_state()
+
+ def depart_math_block(self, node: Element) -> None:
+ self.end_state()
diff --git a/sphinx/writers/xml.py b/sphinx/writers/xml.py
index 47b7357a7..1ae8cc1ab 100644
--- a/sphinx/writers/xml.py
+++ b/sphinx/writers/xml.py
@@ -1,33 +1,52 @@
"""Docutils-native XML and pseudo-XML writers."""
+
from __future__ import annotations
+
from typing import TYPE_CHECKING, Any
+
from docutils.writers.docutils_xml import Writer as BaseXMLWriter
+
if TYPE_CHECKING:
from sphinx.builders import Builder
-class XMLWriter(BaseXMLWriter):
+class XMLWriter(BaseXMLWriter): # type: ignore[misc]
output: str
- def __init__(self, builder: Builder) ->None:
+ def __init__(self, builder: Builder) -> None:
super().__init__()
self.builder = builder
- self.translator_class = (lambda document: self.builder.
- create_translator(document))
+ # A lambda function to generate translator lazily
+ self.translator_class = lambda document: self.builder.create_translator(document)
+
+ def translate(self, *args: Any, **kwargs: Any) -> None:
+ self.document.settings.newlines = \
+ self.document.settings.indents = \
+ self.builder.env.config.xml_pretty
+ self.document.settings.xml_declaration = True
+ self.document.settings.doctype_declaration = True
+ return super().translate()
-class PseudoXMLWriter(BaseXMLWriter):
- supported = 'pprint', 'pformat', 'pseudoxml'
+
+class PseudoXMLWriter(BaseXMLWriter): # type: ignore[misc]
+
+ supported = ('pprint', 'pformat', 'pseudoxml')
"""Formats this writer supports."""
+
config_section = 'pseudoxml writer'
- config_section_dependencies = 'writers',
+ config_section_dependencies = ('writers',)
+
output: str
"""Final translated form of `document`."""
- def __init__(self, builder: Builder) ->None:
+ def __init__(self, builder: Builder) -> None:
super().__init__()
self.builder = builder
- def supports(self, format: str) ->bool:
+ def translate(self) -> None:
+ self.output = self.document.pformat()
+
+ def supports(self, format: str) -> bool:
"""All format-specific elements are supported."""
- pass
+ return True