diff --git a/docs/src/api.rst b/docs/src/api.rst index 7af62fc98..c99706590 100644 --- a/docs/src/api.rst +++ b/docs/src/api.rst @@ -319,7 +319,7 @@ Stage Files ........... To interact with files in your Stage, use the -:attr:`WorkspaceManager.stage` attribute. +:attr:`WorkspaceGroup.stage` attribute. It will return a :class:`Stage` object which defines the following methods and attributes. diff --git a/singlestoredb/config.py b/singlestoredb/config.py index a5e4805de..86d8a8b94 100644 --- a/singlestoredb/config.py +++ b/singlestoredb/config.py @@ -419,6 +419,12 @@ environ=['SINGLESTOREDB_EXT_FUNC_NAME_SUFFIX'], ) +register_option( + 'external_function.function_database', 'string', check_str, '', + 'Database to use for the function definitions.', + environ=['SINGLESTOREDB_EXT_FUNC_FUNCTION_DATABASE'], +) + register_option( 'external_function.connection', 'string', check_str, os.environ.get('SINGLESTOREDB_URL') or None, diff --git a/singlestoredb/docstring/LICENSE.md b/singlestoredb/docstring/LICENSE.md new file mode 100644 index 000000000..f75411dcf --- /dev/null +++ b/singlestoredb/docstring/LICENSE.md @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Marcin Kurczewski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/singlestoredb/docstring/README.md b/singlestoredb/docstring/README.md new file mode 100644 index 000000000..ade85d44b --- /dev/null +++ b/singlestoredb/docstring/README.md @@ -0,0 +1,81 @@ +docstring_parser +================ + +[![Build](https://github.com/rr-/docstring_parser/actions/workflows/build.yml/badge.svg)](https://github.com/rr-/docstring_parser/actions/workflows/build.yml) + +Parse Python docstrings. Currently support ReST, Google, Numpydoc-style and +Epydoc docstrings. + +Example usage: + +```python +>>> from docstring_parser import parse +>>> +>>> +>>> docstring = parse( +... ''' +... Short description +... +... Long description spanning multiple lines +... - First line +... - Second line +... - Third line +... +... :param name: description 1 +... :param int priority: description 2 +... :param str sender: description 3 +... :raises ValueError: if name is invalid +... ''') +>>> +>>> docstring.long_description +'Long description spanning multiple lines\n- First line\n- Second line\n- Third line' +>>> docstring.params[1].arg_name +'priority' +>>> docstring.raises[0].type_name +'ValueError' +``` + +Read [API Documentation](https://rr-.github.io/docstring_parser/). + +# Installation + +Installation using pip + +```shell +pip install docstring_parser + +# or if you want to install it in a virtual environment + +python -m venv venv # create environment +source venv/bin/activate # activate environment +python -m pip install docstring_parser +``` + +Installation using conda + + +1. Download and install miniconda or anaconda +2. Install the package from the conda-forge channel via: + - `conda install -c conda-forge docstring_parser` + - or create a new conda environment via `conda create -n my-new-environment -c conda-forge docstring_parser` + + +# Contributing + +To set up the project: +```sh +git clone https://github.com/rr-/docstring_parser.git +cd docstring_parser + +python -m venv venv # create environment +source venv/bin/activate # activate environment + +pip install -e ".[dev]" # install as editable +pre-commit install # make sure pre-commit is setup +``` + +To run tests: +``` +source venv/bin/activate +pytest +``` diff --git a/singlestoredb/docstring/__init__.py b/singlestoredb/docstring/__init__.py new file mode 100644 index 000000000..6c956da67 --- /dev/null +++ b/singlestoredb/docstring/__init__.py @@ -0,0 +1,33 @@ +"""Parse docstrings as per Sphinx notation.""" +from .common import Docstring +from .common import DocstringDeprecated +from .common import DocstringMeta +from .common import DocstringParam +from .common import DocstringRaises +from .common import DocstringReturns +from .common import DocstringStyle +from .common import ParseError +from .common import RenderingStyle +from .parser import compose +from .parser import parse +from .parser import parse_from_object +from .util import combine_docstrings + +Style = DocstringStyle # backwards compatibility + +__all__ = [ + 'parse', + 'parse_from_object', + 'combine_docstrings', + 'compose', + 'ParseError', + 'Docstring', + 'DocstringMeta', + 'DocstringParam', + 'DocstringRaises', + 'DocstringReturns', + 'DocstringDeprecated', + 'DocstringStyle', + 'RenderingStyle', + 'Style', +] diff --git a/singlestoredb/docstring/attrdoc.py b/singlestoredb/docstring/attrdoc.py new file mode 100644 index 000000000..cad14929a --- /dev/null +++ b/singlestoredb/docstring/attrdoc.py @@ -0,0 +1,126 @@ +"""Attribute docstrings parsing. + +.. seealso:: https://peps.python.org/pep-0257/#what-is-a-docstring +""" +import ast +import inspect +import textwrap +import typing as T +from types import ModuleType + +from .common import Docstring +from .common import DocstringParam + + +def ast_get_constant_value(node: ast.AST) -> T.Any: + """Return the constant's value if the given node is a constant.""" + return getattr(node, 'value') + + +def ast_unparse(node: ast.AST) -> T.Optional[str]: + """Convert the AST node to source code as a string.""" + if hasattr(ast, 'unparse'): + return ast.unparse(node) + # Support simple cases in Python < 3.9 + if isinstance(node, ast.Constant): + return str(ast_get_constant_value(node)) + if isinstance(node, ast.Name): + return node.id + return None + + +def ast_is_literal_str(node: ast.AST) -> bool: + """Return True if the given node is a literal string.""" + return ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(ast_get_constant_value(node.value), str) + ) + + +def ast_get_attribute( + node: ast.AST, +) -> T.Optional[T.Tuple[str, T.Optional[str], T.Optional[str]]]: + """Return name, type and default if the given node is an attribute.""" + if isinstance(node, (ast.Assign, ast.AnnAssign)): + target = ( + node.targets[0] if isinstance(node, ast.Assign) else node.target + ) + if isinstance(target, ast.Name): + type_str = None + if isinstance(node, ast.AnnAssign): + type_str = ast_unparse(node.annotation) + default = None + if node.value: + default = ast_unparse(node.value) + return target.id, type_str, default + return None + + +class AttributeDocstrings(ast.NodeVisitor): + """An ast.NodeVisitor that collects attribute docstrings.""" + + attr_docs: T.Dict[str, T.Tuple[str, T.Optional[str], T.Optional[str]]] = {} + prev_attr = None + + def visit(self, node: T.Any) -> None: + if self.prev_attr and ast_is_literal_str(node): + attr_name, attr_type, attr_default = self.prev_attr + self.attr_docs[attr_name] = ( + ast_get_constant_value(node.value), + attr_type, + attr_default, + ) + self.prev_attr = ast_get_attribute(node) + if isinstance(node, (ast.ClassDef, ast.Module)): + self.generic_visit(node) + + def get_attr_docs( + self, component: T.Any, + ) -> T.Dict[str, T.Tuple[str, T.Optional[str], T.Optional[str]]]: + """Get attribute docstrings from the given component. + + :param component: component to process (class or module) + :returns: for each attribute docstring, a tuple with (description, + type, default) + """ + self.attr_docs = {} + self.prev_attr = None + try: + source = textwrap.dedent(inspect.getsource(component)) + except OSError: + pass + else: + tree = ast.parse(source) + if inspect.ismodule(component): + self.visit(tree) + elif isinstance(tree, ast.Module) and isinstance( + tree.body[0], ast.ClassDef, + ): + self.visit(tree.body[0]) + return self.attr_docs + + +def add_attribute_docstrings( + obj: T.Union[type, ModuleType], docstring: Docstring, +) -> None: + """Add attribute docstrings found in the object's source code. + + :param obj: object from which to parse attribute docstrings + :param docstring: Docstring object where found attributes are added + :returns: list with names of added attributes + """ + params = set(p.arg_name for p in docstring.params) + for arg_name, (description, type_name, default) in ( + AttributeDocstrings().get_attr_docs(obj).items() + ): + if arg_name not in params: + param = DocstringParam( + args=['attribute', arg_name], + description=description, + arg_name=arg_name, + type_name=type_name, + is_optional=default is not None, + default=default, + ) + docstring.meta.append(param) diff --git a/singlestoredb/docstring/common.py b/singlestoredb/docstring/common.py new file mode 100644 index 000000000..a822d39f1 --- /dev/null +++ b/singlestoredb/docstring/common.py @@ -0,0 +1,230 @@ +"""Common methods for parsing.""" +import enum +import typing as T + +PARAM_KEYWORDS = { + 'param', + 'parameter', + 'arg', + 'argument', + 'attribute', + 'key', + 'keyword', +} +RAISES_KEYWORDS = {'raises', 'raise', 'except', 'exception'} +DEPRECATION_KEYWORDS = {'deprecation', 'deprecated'} +RETURNS_KEYWORDS = {'return', 'returns'} +YIELDS_KEYWORDS = {'yield', 'yields'} +EXAMPLES_KEYWORDS = {'example', 'examples'} + + +class ParseError(RuntimeError): + """Base class for all parsing related errors.""" + + +class DocstringStyle(enum.Enum): + """Docstring style.""" + + REST = 1 + GOOGLE = 2 + NUMPYDOC = 3 + EPYDOC = 4 + AUTO = 255 + + +class RenderingStyle(enum.Enum): + """Rendering style when unparsing parsed docstrings.""" + + COMPACT = 1 + CLEAN = 2 + EXPANDED = 3 + + +class DocstringMeta: + """Docstring meta information. + + Symbolizes lines in form of + + :param arg: description + :raises ValueError: if something happens + """ + + def __init__( + self, args: T.List[str], description: T.Optional[str], + ) -> None: + """Initialize self. + + :param args: list of arguments. The exact content of this variable is + dependent on the kind of docstring; it's used to distinguish + between custom docstring meta information items. + :param description: associated docstring description. + """ + self.args = args + self.description = description + + +class DocstringParam(DocstringMeta): + """DocstringMeta symbolizing :param metadata.""" + + def __init__( + self, + args: T.List[str], + description: T.Optional[str], + arg_name: str, + type_name: T.Optional[str], + is_optional: T.Optional[bool], + default: T.Optional[str], + ) -> None: + """Initialize self.""" + super().__init__(args, description) + self.arg_name = arg_name + self.type_name = type_name + self.is_optional = is_optional + self.default = default + + +class DocstringReturns(DocstringMeta): + """DocstringMeta symbolizing :returns or :yields metadata.""" + + def __init__( + self, + args: T.List[str], + description: T.Optional[str], + type_name: T.Optional[str], + is_generator: bool, + return_name: T.Optional[str] = None, + ) -> None: + """Initialize self.""" + super().__init__(args, description) + self.type_name = type_name + self.is_generator = is_generator + self.return_name = return_name + + +class DocstringRaises(DocstringMeta): + """DocstringMeta symbolizing :raises metadata.""" + + def __init__( + self, + args: T.List[str], + description: T.Optional[str], + type_name: T.Optional[str], + ) -> None: + """Initialize self.""" + super().__init__(args, description) + self.type_name = type_name + self.description = description + + +class DocstringDeprecated(DocstringMeta): + """DocstringMeta symbolizing deprecation metadata.""" + + def __init__( + self, + args: T.List[str], + description: T.Optional[str], + version: T.Optional[str], + ) -> None: + """Initialize self.""" + super().__init__(args, description) + self.version = version + self.description = description + + +class DocstringExample(DocstringMeta): + """DocstringMeta symbolizing example metadata.""" + + def __init__( + self, + args: T.List[str], + snippet: T.Optional[str], + description: T.Optional[str], + post_snippet: T.Optional[str] = None, + ) -> None: + """Initialize self.""" + super().__init__(args, description) + self.snippet = snippet + self.description = description + self.post_snippet = post_snippet + + +class Docstring: + """Docstring object representation.""" + + def __init__( + self, + style=None, # type: T.Optional[DocstringStyle] + ) -> None: + """Initialize self.""" + self.short_description = None # type: T.Optional[str] + self.long_description = None # type: T.Optional[str] + self.blank_after_short_description = False + self.blank_after_long_description = False + self.meta = [] # type: T.List[DocstringMeta] + self.style = style # type: T.Optional[DocstringStyle] + + @property + def description(self) -> T.Optional[str]: + """Return the full description of the function + + Returns None if the docstring did not include any description + """ + ret = [] + if self.short_description: + ret.append(self.short_description) + if self.blank_after_short_description: + ret.append('') + if self.long_description: + ret.append(self.long_description) + + if not ret: + return None + + return '\n'.join(ret) + + @property + def params(self) -> T.List[DocstringParam]: + """Return a list of information on function params.""" + return [item for item in self.meta if isinstance(item, DocstringParam)] + + @property + def raises(self) -> T.List[DocstringRaises]: + """Return a list of information on the exceptions that the function + may raise. + """ + return [ + item for item in self.meta if isinstance(item, DocstringRaises) + ] + + @property + def returns(self) -> T.Optional[DocstringReturns]: + """Return a single information on function return. + + Takes the first return information. + """ + for item in self.meta: + if isinstance(item, DocstringReturns): + return item + return None + + @property + def many_returns(self) -> T.List[DocstringReturns]: + """Return a list of information on function return.""" + return [ + item for item in self.meta if isinstance(item, DocstringReturns) + ] + + @property + def deprecation(self) -> T.Optional[DocstringDeprecated]: + """Return a single information on function deprecation notes.""" + for item in self.meta: + if isinstance(item, DocstringDeprecated): + return item + return None + + @property + def examples(self) -> T.List[DocstringExample]: + """Return a list of information on function examples.""" + return [ + item for item in self.meta if isinstance(item, DocstringExample) + ] diff --git a/singlestoredb/docstring/epydoc.py b/singlestoredb/docstring/epydoc.py new file mode 100644 index 000000000..c4bce555c --- /dev/null +++ b/singlestoredb/docstring/epydoc.py @@ -0,0 +1,267 @@ +"""Epyoc-style docstring parsing. + +.. seealso:: http://epydoc.sourceforge.net/manual-fields.html +""" +import inspect +import re +import typing as T + +from .common import Docstring +from .common import DocstringMeta +from .common import DocstringParam +from .common import DocstringRaises +from .common import DocstringReturns +from .common import DocstringStyle +from .common import ParseError +from .common import RenderingStyle + + +def _clean_str(string: str) -> T.Optional[str]: + string = string.strip() + if len(string) > 0: + return string + return None + + +def parse(text: T.Optional[str]) -> Docstring: + """Parse the epydoc-style docstring into its components. + + :returns: parsed docstring + """ + ret = Docstring(style=DocstringStyle.EPYDOC) + if not text: + return ret + + text = inspect.cleandoc(text) + match = re.search('^@', text, flags=re.M) + if match: + desc_chunk = text[:match.start()] + meta_chunk = text[match.start():] + else: + desc_chunk = text + meta_chunk = '' + + parts = desc_chunk.split('\n', 1) + ret.short_description = parts[0] or None + if len(parts) > 1: + long_desc_chunk = parts[1] or '' + ret.blank_after_short_description = long_desc_chunk.startswith('\n') + ret.blank_after_long_description = long_desc_chunk.endswith('\n\n') + ret.long_description = long_desc_chunk.strip() or None + + param_pattern = re.compile( + r'(param|keyword|type)(\s+[_A-z][_A-z0-9]*\??):', + ) + raise_pattern = re.compile(r'(raise)(\s+[_A-z][_A-z0-9]*\??)?:') + return_pattern = re.compile(r'(return|rtype|yield|ytype):') + meta_pattern = re.compile( + r'([_A-z][_A-z0-9]+)((\s+[_A-z][_A-z0-9]*\??)*):', + ) + + # tokenize + stream: T.List[T.Tuple[str, str, T.List[str], str]] = [] + for match in re.finditer( + r'(^@.*?)(?=^@|\Z)', meta_chunk, flags=re.S | re.M, + ): + chunk = match.group(0) + if not chunk: + continue + + param_match = re.search(param_pattern, chunk) + raise_match = re.search(raise_pattern, chunk) + return_match = re.search(return_pattern, chunk) + meta_match = re.search(meta_pattern, chunk) + + match = param_match or raise_match or return_match or meta_match + if not match: + raise ParseError(f'Error parsing meta information near "{chunk}".') + + desc_chunk = chunk[match.end():] + if param_match: + base = 'param' + key = match.group(1) + args = [match.group(2).strip()] + elif raise_match: + base = 'raise' + key = match.group(1) + args = [] if match.group(2) is None else [match.group(2).strip()] + elif return_match: + base = 'return' + key = match.group(1) + args = [] + else: + base = 'meta' + key = match.group(1) + token = _clean_str(match.group(2).strip()) + args = [] if token is None else re.split(r'\s+', token) + + # Make sure we didn't match some existing keyword in an incorrect + # way here: + if key in [ + 'param', + 'keyword', + 'type', + 'return', + 'rtype', + 'yield', + 'ytype', + ]: + raise ParseError( + f'Error parsing meta information near "{chunk}".', + ) + + desc = desc_chunk.strip() + if '\n' in desc: + first_line, rest = desc.split('\n', 1) + desc = first_line + '\n' + inspect.cleandoc(rest) + stream.append((base, key, args, desc)) + + # Combine type_name, arg_name, and description information + params: T.Dict[str, T.Dict[str, T.Any]] = {} + for base, key, args, desc in stream: + if base not in ['param', 'return']: + continue # nothing to do + + (arg_name,) = args or ('return',) + info = params.setdefault(arg_name, {}) + info_key = 'type_name' if 'type' in key else 'description' + info[info_key] = desc + + if base == 'return': + is_generator = key in {'ytype', 'yield'} + if info.setdefault('is_generator', is_generator) != is_generator: + raise ParseError( + f'Error parsing meta information for "{arg_name}".', + ) + + meta_item: T.Union[DocstringParam, DocstringReturns, DocstringRaises, DocstringMeta] + is_done: T.Dict[str, bool] = {} + for base, key, args, desc in stream: + if base == 'param' and not is_done.get(args[0], False): + (arg_name,) = args + info = params[arg_name] + type_name = info.get('type_name') + + if type_name and type_name.endswith('?'): + is_optional = True + type_name = type_name[:-1] + else: + is_optional = False + + match = re.match(r'.*defaults to (.+)', desc, flags=re.DOTALL) + default = match.group(1).rstrip('.') if match else None + + meta_item = DocstringParam( + args=[key, arg_name], + description=info.get('description'), + arg_name=arg_name, + type_name=type_name, + is_optional=is_optional, + default=default, + ) + is_done[arg_name] = True + elif base == 'return' and not is_done.get('return', False): + info = params['return'] + meta_item = DocstringReturns( + args=[key], + description=info.get('description'), + type_name=info.get('type_name'), + is_generator=info.get('is_generator', False), + ) + is_done['return'] = True + elif base == 'raise': + (type_name,) = args or (None,) + meta_item = DocstringRaises( + args=[key] + args, + description=desc, + type_name=type_name, + ) + elif base == 'meta': + meta_item = DocstringMeta( + args=[key] + args, + description=desc, + ) + else: + (key, *_) = args or ('return',) + assert is_done.get(key, False) + continue # don't append + + ret.meta.append(meta_item) + + return ret + + +def compose( + docstring: Docstring, + rendering_style: RenderingStyle = RenderingStyle.COMPACT, + indent: str = ' ', +) -> str: + """Render a parsed docstring into docstring text. + + :param docstring: parsed docstring representation + :param rendering_style: the style to render docstrings + :param indent: the characters used as indentation in the docstring string + :returns: docstring text + """ + + def process_desc(desc: T.Optional[str], is_type: bool) -> str: + if not desc: + return '' + + if rendering_style == RenderingStyle.EXPANDED or ( + rendering_style == RenderingStyle.CLEAN and not is_type + ): + (first, *rest) = desc.splitlines() + return '\n'.join( + ['\n' + indent + first] + [indent + line for line in rest], + ) + + (first, *rest) = desc.splitlines() + return '\n'.join([' ' + first] + [indent + line for line in rest]) + + parts: T.List[str] = [] + if docstring.short_description: + parts.append(docstring.short_description) + if docstring.blank_after_short_description: + parts.append('') + if docstring.long_description: + parts.append(docstring.long_description) + if docstring.blank_after_long_description: + parts.append('') + + for meta in docstring.meta: + if isinstance(meta, DocstringParam): + if meta.type_name: + type_name = ( + f'{meta.type_name}?' + if meta.is_optional + else meta.type_name + ) + text = f'@type {meta.arg_name}:' + text += process_desc(type_name, True) + parts.append(text) + text = f'@param {meta.arg_name}:' + process_desc( + meta.description, False, + ) + parts.append(text) + elif isinstance(meta, DocstringReturns): + (arg_key, type_key) = ( + ('yield', 'ytype') + if meta.is_generator + else ('return', 'rtype') + ) + if meta.type_name: + text = f'@{type_key}:' + process_desc(meta.type_name, True) + parts.append(text) + if meta.description: + text = f'@{arg_key}:' + process_desc(meta.description, False) + parts.append(text) + elif isinstance(meta, DocstringRaises): + text = f'@raise {meta.type_name}:' if meta.type_name else '@raise:' + text += process_desc(meta.description, False) + parts.append(text) + else: + text = f'@{" ".join(meta.args)}:' + text += process_desc(meta.description, False) + parts.append(text) + return '\n'.join(parts) diff --git a/singlestoredb/docstring/google.py b/singlestoredb/docstring/google.py new file mode 100644 index 000000000..2233bf1ff --- /dev/null +++ b/singlestoredb/docstring/google.py @@ -0,0 +1,412 @@ +"""Google-style docstring parsing.""" +import inspect +import re +import typing as T +from collections import namedtuple +from collections import OrderedDict +from enum import IntEnum + +from .common import Docstring +from .common import DocstringExample +from .common import DocstringMeta +from .common import DocstringParam +from .common import DocstringRaises +from .common import DocstringReturns +from .common import DocstringStyle +from .common import EXAMPLES_KEYWORDS +from .common import PARAM_KEYWORDS +from .common import ParseError +from .common import RAISES_KEYWORDS +from .common import RenderingStyle +from .common import RETURNS_KEYWORDS +from .common import YIELDS_KEYWORDS + + +class SectionType(IntEnum): + """Types of sections.""" + + SINGULAR = 0 + """For sections like examples.""" + + MULTIPLE = 1 + """For sections like params.""" + + SINGULAR_OR_MULTIPLE = 2 + """For sections like returns or yields.""" + + +class Section(namedtuple('SectionBase', 'title key type')): + """A docstring section.""" + + +GOOGLE_TYPED_ARG_REGEX = re.compile(r'\s*(.+?)\s*\(\s*(.*[^\s]+)\s*\)') +GOOGLE_ARG_DESC_REGEX = re.compile(r'.*\. Defaults to (.+)\.') +MULTIPLE_PATTERN = re.compile(r'(\s*[^:\s]+:)|([^:]*\]:.*)') + +DEFAULT_SECTIONS = [ + Section('Arguments', 'param', SectionType.MULTIPLE), + Section('Args', 'param', SectionType.MULTIPLE), + Section('Parameters', 'param', SectionType.MULTIPLE), + Section('Params', 'param', SectionType.MULTIPLE), + Section('Raises', 'raises', SectionType.MULTIPLE), + Section('Exceptions', 'raises', SectionType.MULTIPLE), + Section('Except', 'raises', SectionType.MULTIPLE), + Section('Attributes', 'attribute', SectionType.MULTIPLE), + Section('Example', 'examples', SectionType.SINGULAR), + Section('Examples', 'examples', SectionType.SINGULAR), + Section('Returns', 'returns', SectionType.SINGULAR_OR_MULTIPLE), + Section('Yields', 'yields', SectionType.SINGULAR_OR_MULTIPLE), +] + + +class GoogleParser: + """Parser for Google-style docstrings.""" + + def __init__( + self, sections: T.Optional[T.List[Section]] = None, title_colon: bool = True, + ): + """Setup sections. + + :param sections: Recognized sections or None to defaults. + :param title_colon: require colon after section title. + """ + if not sections: + sections = DEFAULT_SECTIONS + self.sections = {s.title: s for s in sections} + self.title_colon = title_colon + self._setup() + + def _setup(self) -> None: + if self.title_colon: + colon = ':' + else: + colon = '' + self.titles_re = re.compile( + '^(' + + '|'.join(f'({t})' for t in self.sections) + + ')' + + colon + + '[ \t\r\f\v]*$', + flags=re.M, + ) + + def _build_meta(self, text: str, title: str) -> DocstringMeta: + """Build docstring element. + + :param text: docstring element text + :param title: title of section containing element + :return: + """ + + section = self.sections[title] + + if ( + section.type == SectionType.SINGULAR_OR_MULTIPLE + and not MULTIPLE_PATTERN.match(text) + ) or section.type == SectionType.SINGULAR: + return self._build_single_meta(section, text) + + if ':' not in text: + raise ParseError(f'Expected a colon in {text!r}.') + + # Split spec and description + before, desc = text.split(':', 1) + + if before and '\n' in before: + # If there is a newline in the first line, clean it up + first_line, rest = before.split('\n', 1) + before = first_line + inspect.cleandoc(rest) + + if desc: + desc = desc[1:] if desc[0] == ' ' else desc + if '\n' in desc: + first_line, rest = desc.split('\n', 1) + desc = first_line + '\n' + inspect.cleandoc(rest) + desc = desc.strip('\n') + + return self._build_multi_meta(section, before, desc) + + @staticmethod + def _build_single_meta(section: Section, desc: str) -> DocstringMeta: + if section.key in RETURNS_KEYWORDS | YIELDS_KEYWORDS: + return DocstringReturns( + args=[section.key], + description=desc, + type_name=None, + is_generator=section.key in YIELDS_KEYWORDS, + ) + if section.key in RAISES_KEYWORDS: + return DocstringRaises( + args=[section.key], description=desc, type_name=None, + ) + if section.key in EXAMPLES_KEYWORDS: + return DocstringExample( + args=[section.key], snippet=None, description=desc, + ) + if section.key in PARAM_KEYWORDS: + raise ParseError('Expected paramenter name.') + return DocstringMeta(args=[section.key], description=desc) + + @staticmethod + def _build_multi_meta( + section: Section, before: str, desc: str, + ) -> DocstringMeta: + if section.key in PARAM_KEYWORDS: + match = GOOGLE_TYPED_ARG_REGEX.match(before) + if match: + arg_name, type_name = match.group(1, 2) + if type_name.endswith(', optional'): + is_optional = True + type_name = type_name[:-10] + elif type_name.endswith('?'): + is_optional = True + type_name = type_name[:-1] + else: + is_optional = False + else: + arg_name, type_name = before, None + is_optional = None + + match = GOOGLE_ARG_DESC_REGEX.match(desc) + default = match.group(1) if match else None + + return DocstringParam( + args=[section.key, before], + description=desc, + arg_name=arg_name, + type_name=type_name, + is_optional=is_optional, + default=default, + ) + if section.key in RETURNS_KEYWORDS | YIELDS_KEYWORDS: + return DocstringReturns( + args=[section.key, before], + description=desc, + type_name=before, + is_generator=section.key in YIELDS_KEYWORDS, + ) + if section.key in RAISES_KEYWORDS: + return DocstringRaises( + args=[section.key, before], description=desc, type_name=before, + ) + return DocstringMeta(args=[section.key, before], description=desc) + + def add_section(self, section: Section) -> None: + """Add or replace a section. + + :param section: The new section. + """ + + self.sections[section.title] = section + self._setup() + + def parse(self, text: T.Optional[str]) -> Docstring: + """Parse the Google-style docstring into its components. + + :returns: parsed docstring + """ + ret = Docstring(style=DocstringStyle.GOOGLE) + if not text: + return ret + + # Clean according to PEP-0257 + text = inspect.cleandoc(text) + + # Find first title and split on its position + match = self.titles_re.search(text) + if match: + desc_chunk = text[:match.start()] + meta_chunk = text[match.start():] + else: + desc_chunk = text + meta_chunk = '' + + # Break description into short and long parts + parts = desc_chunk.split('\n', 1) + ret.short_description = parts[0] or None + if len(parts) > 1: + long_desc_chunk = parts[1] or '' + ret.blank_after_short_description = long_desc_chunk.startswith( + '\n', + ) + ret.blank_after_long_description = long_desc_chunk.endswith('\n\n') + ret.long_description = long_desc_chunk.strip() or None + + # Split by sections determined by titles + matches = list(self.titles_re.finditer(meta_chunk)) + if not matches: + return ret + splits = [] + for j in range(len(matches) - 1): + splits.append((matches[j].end(), matches[j + 1].start())) + splits.append((matches[-1].end(), len(meta_chunk))) + + chunks = OrderedDict() # type: T.MutableMapping[str,str] + for j, (start, end) in enumerate(splits): + title = matches[j].group(1) + if title not in self.sections: + continue + + # Clear Any Unknown Meta + # Ref: https://github.com/rr-/docstring_parser/issues/29 + meta_details = meta_chunk[start:end] + unknown_meta = re.search(r'\n\S', meta_details) + if unknown_meta is not None: + meta_details = meta_details[: unknown_meta.start()] + + chunks[title] = meta_details.strip('\n') + if not chunks: + return ret + + # Add elements from each chunk + for title, chunk in chunks.items(): + # Determine indent + indent_match = re.search(r'^\s*', chunk) + if not indent_match: + raise ParseError(f'Can\'t infer indent from "{chunk}"') + indent = indent_match.group() + + # Check for singular elements + if self.sections[title].type in [ + SectionType.SINGULAR, + SectionType.SINGULAR_OR_MULTIPLE, + ]: + part = inspect.cleandoc(chunk) + ret.meta.append(self._build_meta(part, title)) + continue + + # Split based on lines which have exactly that indent + _re = '^' + indent + r'(?=\S)' + c_matches = list(re.finditer(_re, chunk, flags=re.M)) + if not c_matches: + raise ParseError(f'No specification for "{title}": "{chunk}"') + c_splits = [] + for j in range(len(c_matches) - 1): + c_splits.append((c_matches[j].end(), c_matches[j + 1].start())) + c_splits.append((c_matches[-1].end(), len(chunk))) + for j, (start, end) in enumerate(c_splits): + part = chunk[start:end].strip('\n') + ret.meta.append(self._build_meta(part, title)) + + return ret + + +def parse(text: T.Optional[str]) -> Docstring: + """Parse the Google-style docstring into its components. + + :returns: parsed docstring + """ + return GoogleParser().parse(text) + + +def compose( + docstring: Docstring, + rendering_style: RenderingStyle = RenderingStyle.COMPACT, + indent: str = ' ', +) -> str: + """Render a parsed docstring into docstring text. + + :param docstring: parsed docstring representation + :param rendering_style: the style to render docstrings + :param indent: the characters used as indentation in the docstring string + :returns: docstring text + """ + + def process_one( + one: T.Union[DocstringParam, DocstringReturns, DocstringRaises], + ) -> None: + head = '' + + if isinstance(one, DocstringParam): + head += one.arg_name or '' + elif isinstance(one, DocstringReturns): + head += one.return_name or '' + + if isinstance(one, DocstringParam) and one.is_optional: + optional = ( + '?' + if rendering_style == RenderingStyle.COMPACT + else ', optional' + ) + else: + optional = '' + + if one.type_name and head: + head += f' ({one.type_name}{optional}):' + elif one.type_name: + head += f'{one.type_name}{optional}:' + else: + head += ':' + head = indent + head + + if one.description and rendering_style == RenderingStyle.EXPANDED: + body = f'\n{indent}{indent}'.join( + [head] + one.description.splitlines(), + ) + parts.append(body) + elif one.description: + (first, *rest) = one.description.splitlines() + body = f'\n{indent}{indent}'.join([head + ' ' + first] + rest) + parts.append(body) + else: + parts.append(head) + + def process_sect(name: str, args: T.List[T.Any]) -> None: + if args: + parts.append(name) + for arg in args: + process_one(arg) + parts.append('') + + parts: T.List[str] = [] + if docstring.short_description: + parts.append(docstring.short_description) + if docstring.blank_after_short_description: + parts.append('') + + if docstring.long_description: + parts.append(docstring.long_description) + if docstring.blank_after_long_description: + parts.append('') + + process_sect( + 'Args:', [p for p in docstring.params or [] if p.args[0] == 'param'], + ) + + process_sect( + 'Attributes:', + [p for p in docstring.params or [] if p.args[0] == 'attribute'], + ) + + process_sect( + 'Returns:', + [p for p in docstring.many_returns or [] if not p.is_generator], + ) + + process_sect( + 'Yields:', [p for p in docstring.many_returns or [] if p.is_generator], + ) + + process_sect('Raises:', docstring.raises or []) + + if docstring.returns and not docstring.many_returns: + ret = docstring.returns + parts.append('Yields:' if ret else 'Returns:') + parts.append('-' * len(parts[-1])) + process_one(ret) + + for meta in docstring.meta: + if isinstance( + meta, (DocstringParam, DocstringReturns, DocstringRaises), + ): + continue # Already handled + parts.append(meta.args[0].replace('_', '').title() + ':') + if meta.description: + lines = [indent + m for m in meta.description.splitlines()] + parts.append('\n'.join(lines)) + parts.append('') + + while parts and not parts[-1]: + parts.pop() + + return '\n'.join(parts) diff --git a/singlestoredb/docstring/numpydoc.py b/singlestoredb/docstring/numpydoc.py new file mode 100644 index 000000000..8101950d1 --- /dev/null +++ b/singlestoredb/docstring/numpydoc.py @@ -0,0 +1,562 @@ +"""Numpydoc-style docstring parsing. + +:see: https://numpydoc.readthedocs.io/en/latest/format.html +""" +import inspect +import itertools +import re +import typing as T +from abc import abstractmethod +from textwrap import dedent + +from .common import Docstring +from .common import DocstringDeprecated +from .common import DocstringExample +from .common import DocstringMeta +from .common import DocstringParam +from .common import DocstringRaises +from .common import DocstringReturns +from .common import DocstringStyle +from .common import RenderingStyle + + +def _pairwise( + iterable: T.Iterable[T.Any], + end: T.Optional[T.Any] = None, +) -> T.Iterable[T.Tuple[T.Any, T.Any]]: + left, right = itertools.tee(iterable) + next(right, None) + return itertools.zip_longest(left, right, fillvalue=end) + + +def _clean_str(string: str) -> T.Optional[str]: + string = string.strip() + if len(string) > 0: + return string + return None + + +KV_REGEX = re.compile(r'^[^\s].*$', flags=re.M) +PARAM_KEY_REGEX = re.compile(r'^(?P.*?)(?:\s*:\s*(?P.*?))?$') +PARAM_OPTIONAL_REGEX = re.compile(r'(?P.*?)(?:, optional|\(optional\))$') + +# numpydoc format has no formal grammar for this, +# but we can make some educated guesses... +PARAM_DEFAULT_REGEX = re.compile( + r'(?[\w\-\.]*\w)', +) + +RETURN_KEY_REGEX = re.compile(r'^(?:(?P.*?)\s*:\s*)?(?P.*?)$') + + +class Section: + """Numpydoc section parser. + + :param title: section title. For most sections, this is a heading like + "Parameters" which appears on its own line, underlined by + en-dashes ('-') on the following line. + :param key: meta key string. In the parsed ``DocstringMeta`` instance this + will be the first element of the ``args`` attribute list. + """ + + def __init__(self, title: str, key: str) -> None: + self.title = title + self.key = key + + @property + def title_pattern(self) -> str: + """Regular expression pattern matching this section's header. + + This pattern will match this instance's ``title`` attribute in + an anonymous group. + """ + dashes = '-' * len(self.title) + return rf'^({self.title})\s*?\n{dashes}\s*$' + + def parse(self, text: str) -> T.Iterable[DocstringMeta]: + """Parse ``DocstringMeta`` objects from the body of this section. + + :param text: section body text. Should be cleaned with + ``inspect.cleandoc`` before parsing. + """ + yield DocstringMeta([self.key], description=_clean_str(text)) + + +class _KVSection(Section): + """Base parser for numpydoc sections with key-value syntax. + + E.g. sections that look like this: + key + value + key2 : type + values can also span... + ... multiple lines + """ + + @abstractmethod + def _parse_item(self, key: str, value: str) -> DocstringMeta: + return DocstringMeta(args=[key], description=_clean_str(value)) + + def parse(self, text: str) -> T.Iterable[DocstringMeta]: + for match, next_match in _pairwise(KV_REGEX.finditer(text)): + start = match.end() + end = next_match.start() if next_match is not None else None + value = text[start:end] + yield self._parse_item( + key=match.group(), value=inspect.cleandoc(value), + ) + + +class _SphinxSection(Section): + """Base parser for numpydoc sections with sphinx-style syntax. + + E.g. sections that look like this: + .. title:: something + possibly over multiple lines + """ + + @property + def title_pattern(self) -> str: + return rf'^\.\.\s*({self.title})\s*::' + + +class ParamSection(_KVSection): + """Parser for numpydoc parameter sections. + + E.g. any section that looks like this: + arg_name + arg_description + arg_2 : type, optional + descriptions can also span... + ... multiple lines + """ + + def _parse_item(self, key: str, value: str) -> DocstringParam: + match = PARAM_KEY_REGEX.match(key) + arg_name = type_name = is_optional = None + if match is not None: + arg_name = match.group('name') + type_name = match.group('type') + if type_name is not None: + optional_match = PARAM_OPTIONAL_REGEX.match(type_name) + if optional_match is not None: + type_name = optional_match.group('type') + is_optional = True + else: + is_optional = False + + default = None + if len(value) > 0: + default_match = PARAM_DEFAULT_REGEX.search(value) + if default_match is not None: + default = default_match.group('value') + + return DocstringParam( + args=[self.key, str(arg_name)], + description=_clean_str(value), + arg_name=str(arg_name), + type_name=type_name, + is_optional=is_optional, + default=default, + ) + + +class RaisesSection(_KVSection): + """Parser for numpydoc raises sections. + + E.g. any section that looks like this: + ValueError + A description of what might raise ValueError + """ + + def _parse_item(self, key: str, value: str) -> DocstringRaises: + return DocstringRaises( + args=[self.key, key], + description=_clean_str(value), + type_name=key if len(key) > 0 else None, + ) + + +class ReturnsSection(_KVSection): + """Parser for numpydoc returns sections. + + E.g. any section that looks like this: + return_name : type + A description of this returned value + another_type + Return names are optional, types are required + """ + + is_generator = False + + def _parse_item(self, key: str, value: str) -> DocstringReturns: + match = RETURN_KEY_REGEX.match(key) + if match is not None: + return_name = match.group('name') + type_name = match.group('type') + else: + return_name = None + type_name = None + + return DocstringReturns( + args=[self.key], + description=_clean_str(value), + type_name=type_name, + is_generator=self.is_generator, + return_name=return_name, + ) + + +class YieldsSection(ReturnsSection): + """Parser for numpydoc generator "yields" sections.""" + + is_generator = True + + +class DeprecationSection(_SphinxSection): + """Parser for numpydoc "deprecation warning" sections.""" + + def parse(self, text: str) -> T.Iterable[DocstringDeprecated]: + version, desc, *_ = text.split(sep='\n', maxsplit=1) + [None, None] + + if desc is not None: + desc = _clean_str(inspect.cleandoc(desc)) + + yield DocstringDeprecated( + args=[self.key], description=desc, version=_clean_str(str(version)), + ) + + +class ExamplesSection(Section): + """Parser for numpydoc examples sections. + + E.g. any section that looks like this: + >>> import numpy.matlib + >>> np.matlib.empty((2, 2)) # filled with random data + matrix([[ 6.76425276e-320, 9.79033856e-307], # random + [ 7.39337286e-309, 3.22135945e-309]]) + >>> np.matlib.empty((2, 2), dtype=int) + matrix([[ 6600475, 0], # random + [ 6586976, 22740995]]) + """ + + def parse(self, text: str) -> T.Iterable[DocstringExample]: + """Parse ``DocstringExample`` objects from the body of this section. + + :param text: section body text. Should be cleaned with + ``inspect.cleandoc`` before parsing. + """ + lines = [x.rstrip() for x in dedent(text).strip().splitlines()] + while lines: + snippet_lines = [] + description_lines = [] + post_snippet_lines = [] + + # Parse description of snippet + while lines: + if re.match(r'^(>>>|sql>) ', lines[0]): + break + description_lines.append(lines.pop(0)) + + # Parse code of snippet + while lines: + if not re.match(r'^(>>>|sql>|\.\.\.) ', lines[0]): + break + snippet_lines.append(lines.pop(0)) + + # Parse output of snippet + while lines: + # Bail out at blank lines + if not lines[0]: + lines.pop(0) + break + # Bail out if a new snippet is started + elif re.match(r'^(>>>|sql>) ', lines[0]): + break + else: + snippet_lines.append(lines.pop(0)) + + # if there is following text, but no more snippets, + # make this a post description. + if not [x for x in lines if re.match(r'^(>>>|sql>) ', x)]: + post_snippet_lines.extend(lines) + lines = [] + + yield DocstringExample( + [self.key], + snippet='\n'.join(snippet_lines).strip() if snippet_lines else None, + description='\n'.join(description_lines).strip(), + post_snippet='\n'.join(post_snippet_lines).strip(), + ) + + +DEFAULT_SECTIONS = [ + ParamSection('Parameters', 'param'), + ParamSection('Params', 'param'), + ParamSection('Arguments', 'param'), + ParamSection('Args', 'param'), + ParamSection('Other Parameters', 'other_param'), + ParamSection('Other Params', 'other_param'), + ParamSection('Other Arguments', 'other_param'), + ParamSection('Other Args', 'other_param'), + ParamSection('Receives', 'receives'), + ParamSection('Receive', 'receives'), + RaisesSection('Raises', 'raises'), + RaisesSection('Raise', 'raises'), + RaisesSection('Warns', 'warns'), + RaisesSection('Warn', 'warns'), + ParamSection('Attributes', 'attribute'), + ParamSection('Attribute', 'attribute'), + ReturnsSection('Returns', 'returns'), + ReturnsSection('Return', 'returns'), + YieldsSection('Yields', 'yields'), + YieldsSection('Yield', 'yields'), + ExamplesSection('Examples', 'examples'), + ExamplesSection('Example', 'examples'), + Section('Warnings', 'warnings'), + Section('Warning', 'warnings'), + Section('See Also', 'see_also'), + Section('Related', 'see_also'), + Section('Notes', 'notes'), + Section('Note', 'notes'), + Section('References', 'references'), + Section('Reference', 'references'), + DeprecationSection('deprecated', 'deprecation'), +] + + +class NumpydocParser: + """Parser for numpydoc-style docstrings.""" + + def __init__(self, sections: T.Optional[T.List[Section]] = None): + """Setup sections. + + :param sections: Recognized sections or None to defaults. + """ + sects = sections or DEFAULT_SECTIONS + self.sections = {s.title: s for s in sects} + self._setup() + + def _setup(self) -> None: + self.titles_re = re.compile( + r'|'.join(s.title_pattern for s in self.sections.values()), + flags=re.M, + ) + + def add_section(self, section: Section) -> None: + """Add or replace a section. + + :param section: The new section. + """ + + self.sections[section.title] = section + self._setup() + + def parse(self, text: T.Optional[str]) -> Docstring: + """Parse the numpy-style docstring into its components. + + :returns: parsed docstring + """ + ret = Docstring(style=DocstringStyle.NUMPYDOC) + if not text: + return ret + + # Clean according to PEP-0257 + text = inspect.cleandoc(text) + + # Find first title and split on its position + match = self.titles_re.search(text) + if match: + desc_chunk = text[:match.start()] + meta_chunk = text[match.start():] + else: + desc_chunk = text + meta_chunk = '' + + # Break description into short and long parts + parts = desc_chunk.split('\n', 1) + ret.short_description = parts[0] or None + if len(parts) > 1: + long_desc_chunk = parts[1] or '' + ret.blank_after_short_description = long_desc_chunk.startswith( + '\n', + ) + ret.blank_after_long_description = long_desc_chunk.endswith('\n\n') + ret.long_description = long_desc_chunk.strip() or None + + for match, nextmatch in _pairwise(self.titles_re.finditer(meta_chunk)): + if not match: + raise ValueError( + 'No section title found in docstring: %s' % meta_chunk, + ) + title = next(g for g in match.groups() if g is not None) + factory = self.sections[title] + + # section chunk starts after the header, + # ends at the start of the next header + start = match.end() + end = nextmatch.start() if nextmatch is not None else None + ret.meta.extend(factory.parse(meta_chunk[start:end])) + + return ret + + +def parse(text: T.Optional[str]) -> Docstring: + """Parse the numpy-style docstring into its components. + + :returns: parsed docstring + """ + return NumpydocParser().parse(text) + + +def compose( + # pylint: disable=W0613 + docstring: Docstring, + rendering_style: RenderingStyle = RenderingStyle.COMPACT, + indent: str = ' ', +) -> str: + """Render a parsed docstring into docstring text. + + :param docstring: parsed docstring representation + :param rendering_style: the style to render docstrings + :param indent: the characters used as indentation in the docstring string + :returns: docstring text + """ + + def process_one( + one: T.Union[DocstringParam, DocstringReturns, DocstringRaises], + ) -> None: + head: T.Optional[str] = None + if isinstance(one, DocstringParam): + head = one.arg_name + elif isinstance(one, DocstringReturns): + head = one.return_name + + if one.type_name and head: + head += f' : {one.type_name}' + elif one.type_name: + head = one.type_name + elif not head: + head = '' + + if isinstance(one, DocstringParam) and one.is_optional: + head += ', optional' + + if one.description: + body = f'\n{indent}'.join([head] + one.description.splitlines()) + parts.append(body) + else: + parts.append(head) + + def process_sect(name: str, args: T.List[T.Any]) -> None: + if args: + parts.append('') + parts.append(name) + parts.append('-' * len(parts[-1])) + for arg in args: + process_one(arg) + + parts: T.List[str] = [] + if docstring.short_description: + parts.append(docstring.short_description) + if docstring.blank_after_short_description: + parts.append('') + + if docstring.deprecation: + first = '.. deprecated::' + if docstring.deprecation.version: + first += f' {docstring.deprecation.version}' + if docstring.deprecation.description: + rest = docstring.deprecation.description.splitlines() + else: + rest = [] + sep = f'\n{indent}' + parts.append(sep.join([first] + rest)) + + if docstring.long_description: + parts.append(docstring.long_description) + if docstring.blank_after_long_description: + parts.append('') + + process_sect( + 'Parameters', + [item for item in docstring.params or [] if item.args[0] == 'param'], + ) + + process_sect( + 'Attributes', + [ + item + for item in docstring.params or [] + if item.args[0] == 'attribute' + ], + ) + + process_sect( + 'Returns', + [ + item + for item in docstring.many_returns or [] + if not item.is_generator + ], + ) + + process_sect( + 'Yields', + [item for item in docstring.many_returns or [] if item.is_generator], + ) + + if docstring.returns and not docstring.many_returns: + ret = docstring.returns + parts.append('Yields' if ret else 'Returns') + parts.append('-' * len(parts[-1])) + process_one(ret) + + process_sect( + 'Receives', + [ + item + for item in docstring.params or [] + if item.args[0] == 'receives' + ], + ) + + process_sect( + 'Other Parameters', + [ + item + for item in docstring.params or [] + if item.args[0] == 'other_param' + ], + ) + + process_sect( + 'Raises', + [item for item in docstring.raises or [] if item.args[0] == 'raises'], + ) + + process_sect( + 'Warns', + [item for item in docstring.raises or [] if item.args[0] == 'warns'], + ) + + for meta in docstring.meta: + if isinstance( + meta, + ( + DocstringDeprecated, + DocstringParam, + DocstringReturns, + DocstringRaises, + ), + ): + continue # Already handled + + parts.append('') + parts.append(meta.args[0].replace('_', '').title()) + parts.append('-' * len(meta.args[0])) + + if meta.description: + parts.append(meta.description) + + return '\n'.join(parts) diff --git a/singlestoredb/docstring/parser.py b/singlestoredb/docstring/parser.py new file mode 100644 index 000000000..a5f42400e --- /dev/null +++ b/singlestoredb/docstring/parser.py @@ -0,0 +1,100 @@ +"""The main parsing routine.""" +import inspect +import typing as T + +from . import epydoc +from . import google +from . import numpydoc +from . import rest +from .attrdoc import add_attribute_docstrings +from .common import Docstring +from .common import DocstringStyle +from .common import ParseError +from .common import RenderingStyle + +_STYLE_MAP = { + DocstringStyle.REST: rest, + DocstringStyle.GOOGLE: google, + DocstringStyle.NUMPYDOC: numpydoc, + DocstringStyle.EPYDOC: epydoc, +} + + +def parse( + text: T.Optional[str], style: DocstringStyle = DocstringStyle.AUTO, +) -> Docstring: + """Parse the docstring into its components. + + :param text: docstring text to parse + :param style: docstring style + :returns: parsed docstring representation + """ + if style != DocstringStyle.AUTO: + return _STYLE_MAP[style].parse(text) + + exc: T.Optional[Exception] = None + rets = [] + for module in _STYLE_MAP.values(): + try: + ret = module.parse(text) + except ParseError as ex: + exc = ex + else: + rets.append(ret) + + if not rets and exc is not None: + raise exc + + return sorted(rets, key=lambda d: len(d.meta), reverse=True)[0] + + +def parse_from_object( + obj: T.Any, + style: DocstringStyle = DocstringStyle.AUTO, +) -> Docstring: + """Parse the object's docstring(s) into its components. + + The object can be anything that has a ``__doc__`` attribute. In contrast to + the ``parse`` function, ``parse_from_object`` is able to parse attribute + docstrings which are defined in the source code instead of ``__doc__``. + + Currently only attribute docstrings defined at class and module levels are + supported. Attribute docstrings defined in ``__init__`` methods are not + supported. + + When given a class, only the attribute docstrings of that class are parsed, + not its inherited classes. This is a design decision. Separate calls to + this function should be performed to get attribute docstrings of parent + classes. + + :param obj: object from which to parse the docstring(s) + :param style: docstring style + :returns: parsed docstring representation + """ + docstring = parse(obj.__doc__, style=style) + + if inspect.isclass(obj) or inspect.ismodule(obj): + add_attribute_docstrings(obj, docstring) + + return docstring + + +def compose( + docstring: Docstring, + style: DocstringStyle = DocstringStyle.AUTO, + rendering_style: RenderingStyle = RenderingStyle.COMPACT, + indent: str = ' ', +) -> str: + """Render a parsed docstring into docstring text. + + :param docstring: parsed docstring representation + :param style: docstring style to render + :param indent: the characters used as indentation in the docstring string + :returns: docstring text + """ + st = docstring.style if style == DocstringStyle.AUTO else style + if st is None: + raise ValueError('Docstring style must be specified') + return _STYLE_MAP[st].compose( + docstring, rendering_style=rendering_style, indent=indent, + ) diff --git a/singlestoredb/docstring/py.typed b/singlestoredb/docstring/py.typed new file mode 100644 index 000000000..1242d4327 --- /dev/null +++ b/singlestoredb/docstring/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. diff --git a/singlestoredb/docstring/rest.py b/singlestoredb/docstring/rest.py new file mode 100644 index 000000000..630b6f10e --- /dev/null +++ b/singlestoredb/docstring/rest.py @@ -0,0 +1,256 @@ +"""ReST-style docstring parsing.""" +import inspect +import re +import typing as T + +from .common import DEPRECATION_KEYWORDS +from .common import Docstring +from .common import DocstringDeprecated +from .common import DocstringMeta +from .common import DocstringParam +from .common import DocstringRaises +from .common import DocstringReturns +from .common import DocstringStyle +from .common import PARAM_KEYWORDS +from .common import ParseError +from .common import RAISES_KEYWORDS +from .common import RenderingStyle +from .common import RETURNS_KEYWORDS +from .common import YIELDS_KEYWORDS + + +def _build_meta(args: T.List[str], desc: str) -> DocstringMeta: + key = args[0] + + if key in PARAM_KEYWORDS: + if len(args) == 3: + key, type_name, arg_name = args + if type_name.endswith('?'): + is_optional = True + type_name = type_name[:-1] + else: + is_optional = False + elif len(args) == 2: + key, arg_name = args + type_name = None + is_optional = None + else: + raise ParseError( + f'Expected one or two arguments for a {key} keyword.', + ) + + match = re.match(r'.*defaults to (.+)', desc, flags=re.DOTALL) + default = match.group(1).rstrip('.') if match else None + + return DocstringParam( + args=args, + description=desc, + arg_name=arg_name, + type_name=type_name, + is_optional=is_optional, + default=default, + ) + + if key in RETURNS_KEYWORDS | YIELDS_KEYWORDS: + if len(args) == 2: + type_name = args[1] + elif len(args) == 1: + type_name = None + else: + raise ParseError( + f'Expected one or no arguments for a {key} keyword.', + ) + + return DocstringReturns( + args=args, + description=desc, + type_name=type_name, + is_generator=key in YIELDS_KEYWORDS, + ) + + if key in DEPRECATION_KEYWORDS: + match = re.search( + r'^(?Pv?((?:\d+)(?:\.[0-9a-z\.]+))) (?P.+)', + desc, + flags=re.I, + ) + return DocstringDeprecated( + args=args, + version=match.group('version') if match else None, + description=match.group('desc') if match else desc, + ) + + if key in RAISES_KEYWORDS: + if len(args) == 2: + type_name = args[1] + elif len(args) == 1: + type_name = None + else: + raise ParseError( + f'Expected one or no arguments for a {key} keyword.', + ) + return DocstringRaises( + args=args, description=desc, type_name=type_name, + ) + + return DocstringMeta(args=args, description=desc) + + +def parse(text: T.Optional[str]) -> Docstring: + """Parse the ReST-style docstring into its components. + + :returns: parsed docstring + """ + ret = Docstring(style=DocstringStyle.REST) + if not text: + return ret + + text = inspect.cleandoc(text) + match = re.search('^:', text, flags=re.M) + if match: + desc_chunk = text[:match.start()] + meta_chunk = text[match.start():] + else: + desc_chunk = text + meta_chunk = '' + + parts = desc_chunk.split('\n', 1) + ret.short_description = parts[0] or None + if len(parts) > 1: + long_desc_chunk = parts[1] or '' + ret.blank_after_short_description = long_desc_chunk.startswith('\n') + ret.blank_after_long_description = long_desc_chunk.endswith('\n\n') + ret.long_description = long_desc_chunk.strip() or None + + types = {} + rtypes = {} + for match in re.finditer( + r'(^:.*?)(?=^:|\Z)', meta_chunk, flags=re.S | re.M, + ): + chunk = match.group(0) + if not chunk: + continue + try: + args_chunk, desc_chunk = chunk.lstrip(':').split(':', 1) + except ValueError as ex: + raise ParseError( + f'Error parsing meta information near "{chunk}".', + ) from ex + args = args_chunk.split() + desc = desc_chunk.strip() + + if '\n' in desc: + first_line, rest = desc.split('\n', 1) + desc = first_line + '\n' + inspect.cleandoc(rest) + + # Add special handling for :type a: typename + if len(args) == 2 and args[0] == 'type': + types[args[1]] = desc + elif len(args) in [1, 2] and args[0] == 'rtype': + rtypes[None if len(args) == 1 else args[1]] = desc + else: + ret.meta.append(_build_meta(args, desc)) + + for meta in ret.meta: + if isinstance(meta, DocstringParam): + meta.type_name = meta.type_name or types.get(meta.arg_name) + elif isinstance(meta, DocstringReturns): + meta.type_name = meta.type_name or rtypes.get(meta.return_name) + + if not any(isinstance(m, DocstringReturns) for m in ret.meta) and rtypes: + for return_name, type_name in rtypes.items(): + ret.meta.append( + DocstringReturns( + args=[], + type_name=type_name, + description=None, + is_generator=False, + return_name=return_name, + ), + ) + + return ret + + +def compose( + docstring: Docstring, + rendering_style: RenderingStyle = RenderingStyle.COMPACT, + indent: str = ' ', +) -> str: + """Render a parsed docstring into docstring text. + + :param docstring: parsed docstring representation + :param rendering_style: the style to render docstrings + :param indent: the characters used as indentation in the docstring string + :returns: docstring text + """ + + def process_desc(desc: T.Optional[str]) -> str: + if not desc: + return '' + + if rendering_style == RenderingStyle.CLEAN: + (first, *rest) = desc.splitlines() + return '\n'.join([' ' + first] + [indent + line for line in rest]) + + if rendering_style == RenderingStyle.EXPANDED: + (first, *rest) = desc.splitlines() + return '\n'.join( + ['\n' + indent + first] + [indent + line for line in rest], + ) + + return ' ' + desc + + parts: T.List[str] = [] + if docstring.short_description: + parts.append(docstring.short_description) + if docstring.blank_after_short_description: + parts.append('') + if docstring.long_description: + parts.append(docstring.long_description) + if docstring.blank_after_long_description: + parts.append('') + + for meta in docstring.meta: + if isinstance(meta, DocstringParam): + if meta.type_name: + type_text = ( + f' {meta.type_name}? ' + if meta.is_optional + else f' {meta.type_name} ' + ) + else: + type_text = ' ' + if rendering_style == RenderingStyle.EXPANDED: + text = f':param {meta.arg_name}:' + text += process_desc(meta.description) + parts.append(text) + if type_text[:-1]: + parts.append(f':type {meta.arg_name}:{type_text[:-1]}') + else: + text = f':param{type_text}{meta.arg_name}:' + text += process_desc(meta.description) + parts.append(text) + elif isinstance(meta, DocstringReturns): + type_text = f' {meta.type_name}' if meta.type_name else '' + key = 'yields' if meta.is_generator else 'returns' + + if rendering_style == RenderingStyle.EXPANDED: + if meta.description: + text = f':{key}:' + text += process_desc(meta.description) + parts.append(text) + if type_text: + parts.append(f':rtype:{type_text}') + else: + text = f':{key}{type_text}:' + text += process_desc(meta.description) + parts.append(text) + elif isinstance(meta, DocstringRaises): + type_text = f' {meta.type_name} ' if meta.type_name else '' + text = f':raises{type_text}:' + process_desc(meta.description) + parts.append(text) + else: + text = f':{" ".join(meta.args)}:' + process_desc(meta.description) + parts.append(text) + return '\n'.join(parts) diff --git a/singlestoredb/docstring/tests/__init__.py b/singlestoredb/docstring/tests/__init__.py new file mode 100644 index 000000000..164665e94 --- /dev/null +++ b/singlestoredb/docstring/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for docstring parser.""" diff --git a/singlestoredb/docstring/tests/_pydoctor.py b/singlestoredb/docstring/tests/_pydoctor.py new file mode 100644 index 000000000..a9fb718e9 --- /dev/null +++ b/singlestoredb/docstring/tests/_pydoctor.py @@ -0,0 +1,21 @@ +"""Private pydoctor customization code in order to exclude the package +singlestoredb.docstring.tests from the API documentation. Based on Twisted code. +""" +# pylint: disable=invalid-name + +try: + from pydoctor.model import Documentable, PrivacyClass, System +except ImportError: + pass +else: + + class HidesTestsPydoctorSystem(System): + """A PyDoctor "system" used to generate the docs.""" + + def privacyClass(self, documentable: Documentable) -> PrivacyClass: + """Report the privacy level for an object. Hide the module + 'singlestoredb.docstring.tests'. + """ + if documentable.fullName().startswith('singlestoredb.docstring.tests'): + return PrivacyClass.HIDDEN + return super().privacyClass(documentable) diff --git a/singlestoredb/docstring/tests/test_epydoc.py b/singlestoredb/docstring/tests/test_epydoc.py new file mode 100644 index 000000000..73036dc4f --- /dev/null +++ b/singlestoredb/docstring/tests/test_epydoc.py @@ -0,0 +1,729 @@ +"""Tests for epydoc-style docstring routines.""" +import typing as T + +import pytest + +from singlestoredb.docstring.common import ParseError +from singlestoredb.docstring.common import RenderingStyle +from singlestoredb.docstring.epydoc import compose +from singlestoredb.docstring.epydoc import parse + + +@pytest.mark.parametrize( + 'source, expected', + [ + pytest.param(None, None, id='No __doc__'), + ('', None), + ('\n', None), + ('Short description', 'Short description'), + ('\nShort description\n', 'Short description'), + ('\n Short description\n', 'Short description'), + ], +) +def test_short_description( + source: T.Optional[str], expected: T.Optional[str], +) -> None: + """Test parsing short description.""" + docstring = parse(source) + assert docstring.short_description == expected + assert docstring.long_description is None + assert not docstring.meta + + +@pytest.mark.parametrize( + 'source, expected_short_desc, expected_long_desc, expected_blank', + [ + ( + 'Short description\n\nLong description', + 'Short description', + 'Long description', + True, + ), + ( + """ + Short description + + Long description + """, + 'Short description', + 'Long description', + True, + ), + ( + """ + Short description + + Long description + Second line + """, + 'Short description', + 'Long description\nSecond line', + True, + ), + ( + 'Short description\nLong description', + 'Short description', + 'Long description', + False, + ), + ( + """ + Short description + Long description + """, + 'Short description', + 'Long description', + False, + ), + ( + '\nShort description\nLong description\n', + 'Short description', + 'Long description', + False, + ), + ( + """ + Short description + Long description + Second line + """, + 'Short description', + 'Long description\nSecond line', + False, + ), + ], +) +def test_long_description( + source: str, + expected_short_desc: str, + expected_long_desc: str, + expected_blank: bool, +) -> None: + """Test parsing long description.""" + docstring = parse(source) + assert docstring.short_description == expected_short_desc + assert docstring.long_description == expected_long_desc + assert docstring.blank_after_short_description == expected_blank + assert not docstring.meta + + +@pytest.mark.parametrize( + 'source, expected_short_desc, expected_long_desc, ' + 'expected_blank_short_desc, expected_blank_long_desc', + [ + ( + """ + Short description + @meta: asd + """, + 'Short description', + None, + False, + False, + ), + ( + """ + Short description + Long description + @meta: asd + """, + 'Short description', + 'Long description', + False, + False, + ), + ( + """ + Short description + First line + Second line + @meta: asd + """, + 'Short description', + 'First line\n Second line', + False, + False, + ), + ( + """ + Short description + + First line + Second line + @meta: asd + """, + 'Short description', + 'First line\n Second line', + True, + False, + ), + ( + """ + Short description + + First line + Second line + + @meta: asd + """, + 'Short description', + 'First line\n Second line', + True, + True, + ), + ( + """ + @meta: asd + """, + None, + None, + False, + False, + ), + ], +) +def test_meta_newlines( + source: str, + expected_short_desc: T.Optional[str], + expected_long_desc: T.Optional[str], + expected_blank_short_desc: bool, + expected_blank_long_desc: bool, +) -> None: + """Test parsing newlines around description sections.""" + docstring = parse(source) + assert docstring.short_description == expected_short_desc + assert docstring.long_description == expected_long_desc + assert docstring.blank_after_short_description == expected_blank_short_desc + assert docstring.blank_after_long_description == expected_blank_long_desc + assert len(docstring.meta) == 1 + + +def test_meta_with_multiline_description() -> None: + """Test parsing multiline meta documentation.""" + docstring = parse( + """ + Short description + + @meta: asd + 1 + 2 + 3 + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 1 + assert docstring.meta[0].args == ['meta'] + assert docstring.meta[0].description == 'asd\n1\n 2\n3' + + +def test_multiple_meta() -> None: + """Test parsing multiple meta.""" + docstring = parse( + """ + Short description + + @meta1: asd + 1 + 2 + 3 + @meta2: herp + @meta3: derp + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 3 + assert docstring.meta[0].args == ['meta1'] + assert docstring.meta[0].description == 'asd\n1\n 2\n3' + assert docstring.meta[1].args == ['meta2'] + assert docstring.meta[1].description == 'herp' + assert docstring.meta[2].args == ['meta3'] + assert docstring.meta[2].description == 'derp' + + +def test_meta_with_args() -> None: + """Test parsing meta with additional arguments.""" + docstring = parse( + """ + Short description + + @meta ene due rabe: asd + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 1 + assert docstring.meta[0].args == ['meta', 'ene', 'due', 'rabe'] + assert docstring.meta[0].description == 'asd' + + +def test_params() -> None: + """Test parsing params.""" + docstring = parse('Short description') + assert len(docstring.params) == 0 + + docstring = parse( + """ + Short description + + @param name: description 1 + @param priority: description 2 + @type priority: int + @param sender: description 3 + @type sender: str? + @param message: description 4, defaults to 'hello' + @type message: str? + @param multiline: long description 5, + defaults to 'bye' + @type multiline: str? + """, + ) + assert len(docstring.params) == 5 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'description 1' + assert docstring.params[0].default is None + assert not docstring.params[0].is_optional + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + assert not docstring.params[1].is_optional + assert docstring.params[1].default is None + assert docstring.params[2].arg_name == 'sender' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description == 'description 3' + assert docstring.params[2].is_optional + assert docstring.params[2].default is None + assert docstring.params[3].arg_name == 'message' + assert docstring.params[3].type_name == 'str' + assert ( + docstring.params[3].description == "description 4, defaults to 'hello'" + ) + assert docstring.params[3].is_optional + assert docstring.params[3].default == "'hello'" + assert docstring.params[4].arg_name == 'multiline' + assert docstring.params[4].type_name == 'str' + assert ( + docstring.params[4].description + == "long description 5,\ndefaults to 'bye'" + ) + assert docstring.params[4].is_optional + assert docstring.params[4].default == "'bye'" + + +def test_returns() -> None: + """Test parsing returns.""" + docstring = parse( + """ + Short description + """, + ) + assert docstring.returns is None + + docstring = parse( + """ + Short description + @return: description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name is None + assert docstring.returns.description == 'description' + assert not docstring.returns.is_generator + + docstring = parse( + """ + Short description + @return: description + @rtype: int + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == 'description' + assert not docstring.returns.is_generator + + +def test_yields() -> None: + """Test parsing yields.""" + docstring = parse( + """ + Short description + """, + ) + assert docstring.returns is None + + docstring = parse( + """ + Short description + @yield: description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name is None + assert docstring.returns.description == 'description' + assert docstring.returns.is_generator + + docstring = parse( + """ + Short description + @yield: description + @ytype: int + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == 'description' + assert docstring.returns.is_generator + + +def test_raises() -> None: + """Test parsing raises.""" + docstring = parse( + """ + Short description + """, + ) + assert len(docstring.raises) == 0 + + docstring = parse( + """ + Short description + @raise: description + """, + ) + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name is None + assert docstring.raises[0].description == 'description' + + docstring = parse( + """ + Short description + @raise ValueError: description + """, + ) + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name == 'ValueError' + assert docstring.raises[0].description == 'description' + + +def test_broken_meta() -> None: + """Test parsing broken meta.""" + with pytest.raises(ParseError): + parse('@') + + with pytest.raises(ParseError): + parse('@param herp derp') + + with pytest.raises(ParseError): + parse('@param: invalid') + + with pytest.raises(ParseError): + parse('@param with too many args: desc') + + # these should not raise any errors + parse('@sthstrange: desc') + + +@pytest.mark.parametrize( + 'source, expected', + [ + ('', ''), + ('\n', ''), + ('Short description', 'Short description'), + ('\nShort description\n', 'Short description'), + ('\n Short description\n', 'Short description'), + ( + 'Short description\n\nLong description', + 'Short description\n\nLong description', + ), + ( + """ + Short description + + Long description + """, + 'Short description\n\nLong description', + ), + ( + """ + Short description + + Long description + Second line + """, + 'Short description\n\nLong description\nSecond line', + ), + ( + 'Short description\nLong description', + 'Short description\nLong description', + ), + ( + """ + Short description + Long description + """, + 'Short description\nLong description', + ), + ( + '\nShort description\nLong description\n', + 'Short description\nLong description', + ), + ( + """ + Short description + Long description + Second line + """, + 'Short description\nLong description\nSecond line', + ), + ( + """ + Short description + @meta: asd + """, + 'Short description\n@meta: asd', + ), + ( + """ + Short description + Long description + @meta: asd + """, + 'Short description\nLong description\n@meta: asd', + ), + ( + """ + Short description + First line + Second line + @meta: asd + """, + 'Short description\nFirst line\n Second line\n@meta: asd', + ), + ( + """ + Short description + + First line + Second line + @meta: asd + """, + 'Short description\n' + '\n' + 'First line\n' + ' Second line\n' + '@meta: asd', + ), + ( + """ + Short description + + First line + Second line + + @meta: asd + """, + 'Short description\n' + '\n' + 'First line\n' + ' Second line\n' + '\n' + '@meta: asd', + ), + ( + """ + @meta: asd + """, + '@meta: asd', + ), + ( + """ + Short description + + @meta: asd + 1 + 2 + 3 + """, + 'Short description\n' + '\n' + '@meta: asd\n' + ' 1\n' + ' 2\n' + ' 3', + ), + ( + """ + Short description + + @meta1: asd + 1 + 2 + 3 + @meta2: herp + @meta3: derp + """, + 'Short description\n' + '\n@meta1: asd\n' + ' 1\n' + ' 2\n' + ' 3\n@meta2: herp\n' + '@meta3: derp', + ), + ( + """ + Short description + + @meta ene due rabe: asd + """, + 'Short description\n\n@meta ene due rabe: asd', + ), + ( + """ + Short description + + @param name: description 1 + @param priority: description 2 + @type priority: int + @param sender: description 3 + @type sender: str? + @type message: str? + @param message: description 4, defaults to 'hello' + @type multiline: str? + @param multiline: long description 5, + defaults to 'bye' + """, + 'Short description\n' + '\n' + '@param name: description 1\n' + '@type priority: int\n' + '@param priority: description 2\n' + '@type sender: str?\n' + '@param sender: description 3\n' + '@type message: str?\n' + "@param message: description 4, defaults to 'hello'\n" + '@type multiline: str?\n' + '@param multiline: long description 5,\n' + " defaults to 'bye'", + ), + ( + """ + Short description + @raise: description + """, + 'Short description\n@raise: description', + ), + ( + """ + Short description + @raise ValueError: description + """, + 'Short description\n@raise ValueError: description', + ), + ], +) +def test_compose(source: str, expected: str) -> None: + """Test compose in default mode.""" + assert compose(parse(source)) == expected + + +@pytest.mark.parametrize( + 'source, expected', + [ + ( + """ + Short description + + @param name: description 1 + @param priority: description 2 + @type priority: int + @param sender: description 3 + @type sender: str? + @type message: str? + @param message: description 4, defaults to 'hello' + @type multiline: str? + @param multiline: long description 5, + defaults to 'bye' + """, + 'Short description\n' + '\n' + '@param name:\n' + ' description 1\n' + '@type priority: int\n' + '@param priority:\n' + ' description 2\n' + '@type sender: str?\n' + '@param sender:\n' + ' description 3\n' + '@type message: str?\n' + '@param message:\n' + " description 4, defaults to 'hello'\n" + '@type multiline: str?\n' + '@param multiline:\n' + ' long description 5,\n' + " defaults to 'bye'", + ), + ], +) +def test_compose_clean(source: str, expected: str) -> None: + """Test compose in clean mode.""" + assert ( + compose(parse(source), rendering_style=RenderingStyle.CLEAN) + == expected + ) + + +@pytest.mark.parametrize( + 'source, expected', + [ + ( + """ + Short description + + @param name: description 1 + @param priority: description 2 + @type priority: int + @param sender: description 3 + @type sender: str? + @type message: str? + @param message: description 4, defaults to 'hello' + @type multiline: str? + @param multiline: long description 5, + defaults to 'bye' + """, + 'Short description\n' + '\n' + '@param name:\n' + ' description 1\n' + '@type priority:\n' + ' int\n' + '@param priority:\n' + ' description 2\n' + '@type sender:\n' + ' str?\n' + '@param sender:\n' + ' description 3\n' + '@type message:\n' + ' str?\n' + '@param message:\n' + " description 4, defaults to 'hello'\n" + '@type multiline:\n' + ' str?\n' + '@param multiline:\n' + ' long description 5,\n' + " defaults to 'bye'", + ), + ], +) +def test_compose_expanded(source: str, expected: str) -> None: + """Test compose in expanded mode.""" + assert ( + compose(parse(source), rendering_style=RenderingStyle.EXPANDED) + == expected + ) + + +def test_short_rtype() -> None: + """Test abbreviated docstring with only return type information.""" + string = 'Short description.\n\n@rtype: float' + docstring = parse(string) + assert compose(docstring) == string diff --git a/singlestoredb/docstring/tests/test_google.py b/singlestoredb/docstring/tests/test_google.py new file mode 100644 index 000000000..eed228d8b --- /dev/null +++ b/singlestoredb/docstring/tests/test_google.py @@ -0,0 +1,1007 @@ +"""Tests for Google-style docstring routines.""" +import typing as T + +import pytest + +import singlestoredb.docstring.google as google +from singlestoredb.docstring.common import ParseError +from singlestoredb.docstring.common import RenderingStyle +from singlestoredb.docstring.google import compose +from singlestoredb.docstring.google import GoogleParser +from singlestoredb.docstring.google import parse +from singlestoredb.docstring.google import Section +from singlestoredb.docstring.google import SectionType + + +def test_google_parser_unknown_section() -> None: + """Test parsing an unknown section with default GoogleParser + configuration. + """ + parser = GoogleParser() + docstring = parser.parse( + """ + Unknown: + spam: a + """, + ) + assert docstring.short_description == 'Unknown:' + assert docstring.long_description == 'spam: a' + assert len(docstring.meta) == 0 + + +def test_google_parser_multi_line_parameter_type() -> None: + """Test parsing a multi-line parameter type with default GoogleParser""" + parser = GoogleParser() + docstring = parser.parse( + """Description of the function. + + Args: + output_type (Literal["searchResults", "sourcedAnswer", + "structured"]): The type of output. + This can be one of the following: + - "searchResults": Represents the search results. + - "sourcedAnswer": Represents a sourced answer. + - "structured": Represents a structured output format. + + Returns: + bool: Indicates success or failure. + + """, + ) + assert docstring.params[0].arg_name == 'output_type' + + +def test_google_parser_custom_sections() -> None: + """Test parsing an unknown section with custom GoogleParser + configuration. + """ + parser = GoogleParser( + [ + Section('DESCRIPTION', 'desc', SectionType.SINGULAR), + Section('ARGUMENTS', 'param', SectionType.MULTIPLE), + Section('ATTRIBUTES', 'attribute', SectionType.MULTIPLE), + Section('EXAMPLES', 'examples', SectionType.SINGULAR), + ], + title_colon=False, + ) + docstring = parser.parse( + """ + DESCRIPTION + This is the description. + + ARGUMENTS + arg1: first arg + arg2: second arg + + ATTRIBUTES + attr1: first attribute + attr2: second attribute + + EXAMPLES + Many examples + More examples + """, + ) + + assert docstring.short_description is None + assert docstring.long_description is None + assert len(docstring.meta) == 6 + assert docstring.meta[0].args == ['desc'] + assert docstring.meta[0].description == 'This is the description.' + assert docstring.meta[1].args == ['param', 'arg1'] + assert docstring.meta[1].description == 'first arg' + assert docstring.meta[2].args == ['param', 'arg2'] + assert docstring.meta[2].description == 'second arg' + assert docstring.meta[3].args == ['attribute', 'attr1'] + assert docstring.meta[3].description == 'first attribute' + assert docstring.meta[4].args == ['attribute', 'attr2'] + assert docstring.meta[4].description == 'second attribute' + assert docstring.meta[5].args == ['examples'] + assert docstring.meta[5].description == 'Many examples\nMore examples' + + +def test_google_parser_custom_sections_after() -> None: + """Test parsing an unknown section with custom GoogleParser configuration + that was set at a runtime. + """ + parser = GoogleParser(title_colon=False) + parser.add_section(Section('Note', 'note', SectionType.SINGULAR)) + docstring = parser.parse( + """ + short description + + Note: + a note + """, + ) + assert docstring.short_description == 'short description' + assert docstring.long_description == 'Note:\n a note' + + docstring = parser.parse( + """ + short description + + Note a note + """, + ) + assert docstring.short_description == 'short description' + assert docstring.long_description == 'Note a note' + + docstring = parser.parse( + """ + short description + + Note + a note + """, + ) + assert len(docstring.meta) == 1 + assert docstring.meta[0].args == ['note'] + assert docstring.meta[0].description == 'a note' + + +@pytest.mark.parametrize( + 'source, expected', + [ + pytest.param(None, None, id='No __doc__'), + ('', None), + ('\n', None), + ('Short description', 'Short description'), + ('\nShort description\n', 'Short description'), + ('\n Short description\n', 'Short description'), + ], +) +def test_short_description( + source: T.Optional[str], expected: T.Optional[str], +) -> None: + """Test parsing short description.""" + docstring = parse(source) + assert docstring.short_description == expected + assert docstring.long_description is None + assert not docstring.meta + + +@pytest.mark.parametrize( + 'source, expected_short_desc, expected_long_desc, expected_blank', + [ + ( + 'Short description\n\nLong description', + 'Short description', + 'Long description', + True, + ), + ( + """ + Short description + + Long description + """, + 'Short description', + 'Long description', + True, + ), + ( + """ + Short description + + Long description + Second line + """, + 'Short description', + 'Long description\nSecond line', + True, + ), + ( + 'Short description\nLong description', + 'Short description', + 'Long description', + False, + ), + ( + """ + Short description + Long description + """, + 'Short description', + 'Long description', + False, + ), + ( + '\nShort description\nLong description\n', + 'Short description', + 'Long description', + False, + ), + ( + """ + Short description + Long description + Second line + """, + 'Short description', + 'Long description\nSecond line', + False, + ), + ], +) +def test_long_description( + source: str, + expected_short_desc: str, + expected_long_desc: str, + expected_blank: bool, +) -> None: + """Test parsing long description.""" + docstring = parse(source) + assert docstring.short_description == expected_short_desc + assert docstring.long_description == expected_long_desc + assert docstring.blank_after_short_description == expected_blank + assert not docstring.meta + + +@pytest.mark.parametrize( + 'source, expected_short_desc, expected_long_desc, ' + 'expected_blank_short_desc, expected_blank_long_desc', + [ + ( + """ + Short description + Args: + asd: + """, + 'Short description', + None, + False, + False, + ), + ( + """ + Short description + Long description + Args: + asd: + """, + 'Short description', + 'Long description', + False, + False, + ), + ( + """ + Short description + First line + Second line + Args: + asd: + """, + 'Short description', + 'First line\n Second line', + False, + False, + ), + ( + """ + Short description + + First line + Second line + Args: + asd: + """, + 'Short description', + 'First line\n Second line', + True, + False, + ), + ( + """ + Short description + + First line + Second line + + Args: + asd: + """, + 'Short description', + 'First line\n Second line', + True, + True, + ), + ( + """ + Args: + asd: + """, + None, + None, + False, + False, + ), + ], +) +def test_meta_newlines( + source: str, + expected_short_desc: T.Optional[str], + expected_long_desc: T.Optional[str], + expected_blank_short_desc: bool, + expected_blank_long_desc: bool, +) -> None: + """Test parsing newlines around description sections.""" + docstring = parse(source) + assert docstring.short_description == expected_short_desc + assert docstring.long_description == expected_long_desc + assert docstring.blank_after_short_description == expected_blank_short_desc + assert docstring.blank_after_long_description == expected_blank_long_desc + assert len(docstring.meta) == 1 + + +def test_meta_with_multiline_description() -> None: + """Test parsing multiline meta documentation.""" + docstring = parse( + """ + Short description + + Args: + spam: asd + 1 + 2 + 3 + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 1 + assert docstring.meta[0].args == ['param', 'spam'] + assert isinstance(docstring.meta[0], google.DocstringParam) + assert docstring.meta[0].arg_name == 'spam' + assert docstring.meta[0].description == 'asd\n1\n 2\n3' + + +def test_default_args() -> None: + """Test parsing default arguments.""" + docstring = parse( + """A sample function + +A function the demonstrates docstrings + +Args: + arg1 (int): The firsty arg + arg2 (str): The second arg + arg3 (float, optional): The third arg. Defaults to 1.0. + arg4 (Optional[Dict[str, Any]], optional): The last arg. Defaults to None. + arg5 (str, optional): The fifth arg. Defaults to DEFAULT_ARG5. + +Returns: + Mapping[str, Any]: The args packed in a mapping +""", + ) + assert docstring is not None + assert len(docstring.params) == 5 + + arg4 = docstring.params[3] + assert arg4.arg_name == 'arg4' + assert arg4.is_optional + assert arg4.type_name == 'Optional[Dict[str, Any]]' + assert arg4.default == 'None' + assert arg4.description == 'The last arg. Defaults to None.' + + +def test_multiple_meta() -> None: + """Test parsing multiple meta.""" + docstring = parse( + """ + Short description + + Args: + spam: asd + 1 + 2 + 3 + + Raises: + bla: herp + yay: derp + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 3 + assert docstring.meta[0].args == ['param', 'spam'] + assert isinstance(docstring.meta[0], google.DocstringParam) + assert docstring.meta[0].arg_name == 'spam' + assert docstring.meta[0].description == 'asd\n1\n 2\n3' + assert docstring.meta[1].args == ['raises', 'bla'] + assert isinstance(docstring.meta[1], google.DocstringRaises) + assert docstring.meta[1].type_name == 'bla' + assert docstring.meta[1].description == 'herp' + assert isinstance(docstring.meta[2], google.DocstringRaises) + assert docstring.meta[2].args == ['raises', 'yay'] + assert docstring.meta[2].type_name == 'yay' + assert docstring.meta[2].description == 'derp' + + +def test_params() -> None: + """Test parsing params.""" + docstring = parse('Short description') + assert len(docstring.params) == 0 + + docstring = parse( + """ + Short description + + Args: + name: description 1 + priority (int): description 2 + sender (str?): description 3 + ratio (Optional[float], optional): description 4 + """, + ) + assert len(docstring.params) == 4 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'description 1' + assert not docstring.params[0].is_optional + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + assert not docstring.params[1].is_optional + assert docstring.params[2].arg_name == 'sender' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description == 'description 3' + assert docstring.params[2].is_optional + assert docstring.params[3].arg_name == 'ratio' + assert docstring.params[3].type_name == 'Optional[float]' + assert docstring.params[3].description == 'description 4' + assert docstring.params[3].is_optional + + docstring = parse( + """ + Short description + + Args: + name: description 1 + with multi-line text + priority (int): description 2 + """, + ) + assert len(docstring.params) == 2 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == ( + 'description 1\nwith multi-line text' + ) + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + + +def test_attributes() -> None: + """Test parsing attributes.""" + docstring = parse('Short description') + assert len(docstring.params) == 0 + + docstring = parse( + """ + Short description + + Attributes: + name: description 1 + priority (int): description 2 + sender (str?): description 3 + ratio (Optional[float], optional): description 4 + """, + ) + assert len(docstring.params) == 4 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'description 1' + assert not docstring.params[0].is_optional + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + assert not docstring.params[1].is_optional + assert docstring.params[2].arg_name == 'sender' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description == 'description 3' + assert docstring.params[2].is_optional + assert docstring.params[3].arg_name == 'ratio' + assert docstring.params[3].type_name == 'Optional[float]' + assert docstring.params[3].description == 'description 4' + assert docstring.params[3].is_optional + + docstring = parse( + """ + Short description + + Attributes: + name: description 1 + with multi-line text + priority (int): description 2 + """, + ) + assert len(docstring.params) == 2 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == ( + 'description 1\nwith multi-line text' + ) + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + + +def test_returns() -> None: + """Test parsing returns.""" + docstring = parse( + """ + Short description + """, + ) + assert docstring.returns is None + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 0 + + docstring = parse( + """ + Short description + Returns: + description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name is None + assert docstring.returns.description == 'description' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Short description + Returns: + description with: a colon! + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name is None + assert docstring.returns.description == 'description with: a colon!' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Short description + Returns: + int: description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == 'description' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Returns: + Optional[Mapping[str, List[int]]]: A description: with a colon + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'Optional[Mapping[str, List[int]]]' + assert docstring.returns.description == 'A description: with a colon' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Short description + Yields: + int: description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == 'description' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Short description + Returns: + int: description + with much text + + even some spacing + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == ( + 'description\nwith much text\n\neven some spacing' + ) + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + +def test_raises() -> None: + """Test parsing raises.""" + docstring = parse( + """ + Short description + """, + ) + assert len(docstring.raises) == 0 + + docstring = parse( + """ + Short description + Raises: + ValueError: description + """, + ) + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name == 'ValueError' + assert docstring.raises[0].description == 'description' + + +def test_examples() -> None: + """Test parsing examples.""" + docstring = parse( + """ + Short description + Example: + example: 1 + Examples: + long example + + more here + """, + ) + assert len(docstring.examples) == 2 + assert docstring.examples[0].description == 'example: 1' + assert docstring.examples[1].description == 'long example\n\nmore here' + + +def test_broken_meta() -> None: + """Test parsing broken meta.""" + with pytest.raises(ParseError): + parse('Args:') + + with pytest.raises(ParseError): + parse('Args:\n herp derp') + + +def test_unknown_meta() -> None: + """Test parsing unknown meta.""" + docstring = parse( + """Short desc + + Unknown 0: + title0: content0 + + Args: + arg0: desc0 + arg1: desc1 + + Unknown1: + title1: content1 + + Unknown2: + title2: content2 + """, + ) + + assert docstring.params[0].arg_name == 'arg0' + assert docstring.params[0].description == 'desc0' + assert docstring.params[1].arg_name == 'arg1' + assert docstring.params[1].description == 'desc1' + + +def test_broken_arguments() -> None: + """Test parsing broken arguments.""" + with pytest.raises(ParseError): + parse( + """This is a test + + Args: + param - poorly formatted + """, + ) + + +def test_empty_example() -> None: + """Test parsing empty examples section.""" + docstring = parse( + """Short description + + Example: + + Raises: + IOError: some error + """, + ) + + assert len(docstring.examples) == 1 + assert docstring.examples[0].args == ['examples'] + assert docstring.examples[0].description == '' + + +@pytest.mark.parametrize( + 'source, expected', + [ + ('', ''), + ('\n', ''), + ('Short description', 'Short description'), + ('\nShort description\n', 'Short description'), + ('\n Short description\n', 'Short description'), + ( + 'Short description\n\nLong description', + 'Short description\n\nLong description', + ), + ( + """ + Short description + + Long description + """, + 'Short description\n\nLong description', + ), + ( + """ + Short description + + Long description + Second line + """, + 'Short description\n\nLong description\nSecond line', + ), + ( + 'Short description\nLong description', + 'Short description\nLong description', + ), + ( + """ + Short description + Long description + """, + 'Short description\nLong description', + ), + ( + '\nShort description\nLong description\n', + 'Short description\nLong description', + ), + ( + """ + Short description + Long description + Second line + """, + 'Short description\nLong description\nSecond line', + ), + ( + """ + Short description + Meta: + asd + """, + 'Short description\nMeta:\n asd', + ), + ( + """ + Short description + Long description + Meta: + asd + """, + 'Short description\nLong description\nMeta:\n asd', + ), + ( + """ + Short description + First line + Second line + Meta: + asd + """, + 'Short description\n' + 'First line\n' + ' Second line\n' + 'Meta:\n' + ' asd', + ), + ( + """ + Short description + + First line + Second line + Meta: + asd + """, + 'Short description\n' + '\n' + 'First line\n' + ' Second line\n' + 'Meta:\n' + ' asd', + ), + ( + """ + Short description + + First line + Second line + + Meta: + asd + """, + 'Short description\n' + '\n' + 'First line\n' + ' Second line\n' + '\n' + 'Meta:\n' + ' asd', + ), + ( + """ + Short description + + Meta: + asd + 1 + 2 + 3 + """, + 'Short description\n' + '\n' + 'Meta:\n' + ' asd\n' + ' 1\n' + ' 2\n' + ' 3', + ), + ( + """ + Short description + + Meta1: + asd + 1 + 2 + 3 + Meta2: + herp + Meta3: + derp + """, + 'Short description\n' + '\n' + 'Meta1:\n' + ' asd\n' + ' 1\n' + ' 2\n' + ' 3\n' + 'Meta2:\n' + ' herp\n' + 'Meta3:\n' + ' derp', + ), + ( + """ + Short description + + Args: + name: description 1 + priority (int): description 2 + sender (str, optional): description 3 + message (str, optional): description 4, defaults to 'hello' + multiline (str?): + long description 5, + defaults to 'bye' + """, + 'Short description\n' + '\n' + 'Args:\n' + ' name: description 1\n' + ' priority (int): description 2\n' + ' sender (str?): description 3\n' + " message (str?): description 4, defaults to 'hello'\n" + ' multiline (str?): long description 5,\n' + " defaults to 'bye'", + ), + ( + """ + Short description + Raises: + ValueError: description + """, + 'Short description\nRaises:\n ValueError: description', + ), + ], +) +def test_compose(source: str, expected: str) -> None: + """Test compose in default mode.""" + assert compose(parse(source)) == expected + + +@pytest.mark.parametrize( + 'source, expected', + [ + ( + """ + Short description + + Args: + name: description 1 + priority (int): description 2 + sender (str, optional): description 3 + message (str, optional): description 4, defaults to 'hello' + multiline (str?): + long description 5, + defaults to 'bye' + """, + 'Short description\n' + '\n' + 'Args:\n' + ' name: description 1\n' + ' priority (int): description 2\n' + ' sender (str, optional): description 3\n' + " message (str, optional): description 4, defaults to 'hello'\n" + ' multiline (str, optional): long description 5,\n' + " defaults to 'bye'", + ), + ], +) +def test_compose_clean(source: str, expected: str) -> None: + """Test compose in clean mode.""" + assert ( + compose(parse(source), rendering_style=RenderingStyle.CLEAN) + == expected + ) + + +@pytest.mark.parametrize( + 'source, expected', + [ + ( + """ + Short description + + Args: + name: description 1 + priority (int): description 2 + sender (str, optional): description 3 + message (str, optional): description 4, defaults to 'hello' + multiline (str?): + long description 5, + defaults to 'bye' + """, + 'Short description\n' + '\n' + 'Args:\n' + ' name:\n' + ' description 1\n' + ' priority (int):\n' + ' description 2\n' + ' sender (str, optional):\n' + ' description 3\n' + ' message (str, optional):\n' + " description 4, defaults to 'hello'\n" + ' multiline (str, optional):\n' + ' long description 5,\n' + " defaults to 'bye'", + ), + ], +) +def test_compose_expanded(source: str, expected: str) -> None: + """Test compose in expanded mode.""" + assert ( + compose(parse(source), rendering_style=RenderingStyle.EXPANDED) + == expected + ) diff --git a/singlestoredb/docstring/tests/test_numpydoc.py b/singlestoredb/docstring/tests/test_numpydoc.py new file mode 100644 index 000000000..b844eebb1 --- /dev/null +++ b/singlestoredb/docstring/tests/test_numpydoc.py @@ -0,0 +1,1100 @@ +"""Tests for numpydoc-style docstring routines.""" +import typing as T + +import pytest + +import singlestoredb.docstring.numpydoc as numpydoc +from singlestoredb.docstring.numpydoc import compose +from singlestoredb.docstring.numpydoc import parse + + +@pytest.mark.parametrize( + 'source, expected', + [ + pytest.param(None, None, id='No __doc__'), + ('', None), + ('\n', None), + ('Short description', 'Short description'), + ('\nShort description\n', 'Short description'), + ('\n Short description\n', 'Short description'), + ], +) +def test_short_description( + source: T.Optional[str], expected: T.Optional[str], +) -> None: + """Test parsing short description.""" + docstring = parse(source) + assert docstring.short_description == expected + assert docstring.long_description is None + assert not docstring.meta + + +@pytest.mark.parametrize( + 'source, expected_short_desc, expected_long_desc, expected_blank', + [ + ( + 'Short description\n\nLong description', + 'Short description', + 'Long description', + True, + ), + ( + """ + Short description + + Long description + """, + 'Short description', + 'Long description', + True, + ), + ( + """ + Short description + + Long description + Second line + """, + 'Short description', + 'Long description\nSecond line', + True, + ), + ( + 'Short description\nLong description', + 'Short description', + 'Long description', + False, + ), + ( + """ + Short description + Long description + """, + 'Short description', + 'Long description', + False, + ), + ( + '\nShort description\nLong description\n', + 'Short description', + 'Long description', + False, + ), + ( + """ + Short description + Long description + Second line + """, + 'Short description', + 'Long description\nSecond line', + False, + ), + ], +) +def test_long_description( + source: str, + expected_short_desc: str, + expected_long_desc: str, + expected_blank: bool, +) -> None: + """Test parsing long description.""" + docstring = parse(source) + assert docstring.short_description == expected_short_desc + assert docstring.long_description == expected_long_desc + assert docstring.blank_after_short_description == expected_blank + assert not docstring.meta + + +@pytest.mark.parametrize( + 'source, expected_short_desc, expected_long_desc, ' + 'expected_blank_short_desc, expected_blank_long_desc', + [ + ( + """ + Short description + Parameters + ---------- + asd + """, + 'Short description', + None, + False, + False, + ), + ( + """ + Short description + Long description + Parameters + ---------- + asd + """, + 'Short description', + 'Long description', + False, + False, + ), + ( + """ + Short description + First line + Second line + Parameters + ---------- + asd + """, + 'Short description', + 'First line\n Second line', + False, + False, + ), + ( + """ + Short description + + First line + Second line + Parameters + ---------- + asd + """, + 'Short description', + 'First line\n Second line', + True, + False, + ), + ( + """ + Short description + + First line + Second line + + Parameters + ---------- + asd + """, + 'Short description', + 'First line\n Second line', + True, + True, + ), + ( + """ + Parameters + ---------- + asd + """, + None, + None, + False, + False, + ), + ], +) +def test_meta_newlines( + source: str, + expected_short_desc: T.Optional[str], + expected_long_desc: T.Optional[str], + expected_blank_short_desc: bool, + expected_blank_long_desc: bool, +) -> None: + """Test parsing newlines around description sections.""" + docstring = parse(source) + assert docstring.short_description == expected_short_desc + assert docstring.long_description == expected_long_desc + assert docstring.blank_after_short_description == expected_blank_short_desc + assert docstring.blank_after_long_description == expected_blank_long_desc + assert len(docstring.meta) == 1 + + +def test_meta_with_multiline_description() -> None: + """Test parsing multiline meta documentation.""" + docstring = parse( + """ + Short description + + Parameters + ---------- + spam + asd + 1 + 2 + 3 + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 1 + assert docstring.meta[0].args == ['param', 'spam'] + assert isinstance(docstring.meta[0], numpydoc.DocstringParam) + assert docstring.meta[0].arg_name == 'spam' + assert docstring.meta[0].description == 'asd\n1\n 2\n3' + + +@pytest.mark.parametrize( + 'source, expected_is_optional, expected_type_name, expected_default', + [ + ( + """ + Parameters + ---------- + arg1 : int + The first arg + """, + False, + 'int', + None, + ), + ( + """ + Parameters + ---------- + arg2 : str + The second arg + """, + False, + 'str', + None, + ), + ( + """ + Parameters + ---------- + arg3 : float, optional + The third arg. Default is 1.0. + """, + True, + 'float', + '1.0', + ), + ( + """ + Parameters + ---------- + arg4 : Optional[Dict[str, Any]], optional + The fourth arg. Defaults to None + """, + True, + 'Optional[Dict[str, Any]]', + 'None', + ), + ( + """ + Parameters + ---------- + arg5 : str, optional + The fifth arg. Default: DEFAULT_ARGS + """, + True, + 'str', + 'DEFAULT_ARGS', + ), + ( + """ + Parameters + ---------- + parameter_without_default : int + The parameter_without_default is required. + """, + False, + 'int', + None, + ), + ], +) +def test_default_args( + source: str, + expected_is_optional: bool, + expected_type_name: T.Optional[str], + expected_default: T.Optional[str], +) -> None: + """Test parsing default arguments.""" + docstring = parse(source) + assert docstring is not None + assert len(docstring.params) == 1 + + arg1 = docstring.params[0] + assert arg1.is_optional == expected_is_optional + assert arg1.type_name == expected_type_name + assert arg1.default == expected_default + + +def test_multiple_meta() -> None: + """Test parsing multiple meta.""" + docstring = parse( + """ + Short description + + Parameters + ---------- + spam + asd + 1 + 2 + 3 + + Raises + ------ + bla + herp + yay + derp + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 3 + assert docstring.meta[0].args == ['param', 'spam'] + assert isinstance(docstring.meta[0], numpydoc.DocstringParam) + assert docstring.meta[0].arg_name == 'spam' + assert docstring.meta[0].description == 'asd\n1\n 2\n3' + assert docstring.meta[1].args == ['raises', 'bla'] + assert isinstance(docstring.meta[1], numpydoc.DocstringRaises) + assert docstring.meta[1].type_name == 'bla' + assert docstring.meta[1].description == 'herp' + assert docstring.meta[2].args == ['raises', 'yay'] + assert isinstance(docstring.meta[2], numpydoc.DocstringRaises) + assert docstring.meta[2].type_name == 'yay' + assert docstring.meta[2].description == 'derp' + + +def test_params() -> None: + """Test parsing params.""" + docstring = parse('Short description') + assert len(docstring.params) == 0 + + docstring = parse( + """ + Short description + + Parameters + ---------- + name + description 1 + priority : int + description 2 + sender : str, optional + description 3 + ratio : Optional[float], optional + description 4 + """, + ) + assert len(docstring.params) == 4 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'description 1' + assert not docstring.params[0].is_optional + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + assert not docstring.params[1].is_optional + assert docstring.params[2].arg_name == 'sender' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description == 'description 3' + assert docstring.params[2].is_optional + assert docstring.params[3].arg_name == 'ratio' + assert docstring.params[3].type_name == 'Optional[float]' + assert docstring.params[3].description == 'description 4' + assert docstring.params[3].is_optional + + docstring = parse( + """ + Short description + + Parameters + ---------- + name + description 1 + with multi-line text + priority : int + description 2 + """, + ) + assert len(docstring.params) == 2 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == ( + 'description 1\nwith multi-line text' + ) + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + + +def test_attributes() -> None: + """Test parsing attributes.""" + docstring = parse('Short description') + assert len(docstring.params) == 0 + + docstring = parse( + """ + Short description + + Attributes + ---------- + name + description 1 + priority : int + description 2 + sender : str, optional + description 3 + ratio : Optional[float], optional + description 4 + """, + ) + assert len(docstring.params) == 4 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'description 1' + assert not docstring.params[0].is_optional + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + assert not docstring.params[1].is_optional + assert docstring.params[2].arg_name == 'sender' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description == 'description 3' + assert docstring.params[2].is_optional + assert docstring.params[3].arg_name == 'ratio' + assert docstring.params[3].type_name == 'Optional[float]' + assert docstring.params[3].description == 'description 4' + assert docstring.params[3].is_optional + + docstring = parse( + """ + Short description + + Attributes + ---------- + name + description 1 + with multi-line text + priority : int + description 2 + """, + ) + assert len(docstring.params) == 2 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == ( + 'description 1\nwith multi-line text' + ) + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + + +def test_other_params() -> None: + """Test parsing other parameters.""" + docstring = parse( + """ + Short description + Other Parameters + ---------------- + only_seldom_used_keywords : type, optional + Explanation + common_parameters_listed_above : type, optional + Explanation + """, + ) + assert len(docstring.meta) == 2 + assert docstring.meta[0].args == [ + 'other_param', + 'only_seldom_used_keywords', + ] + assert isinstance(docstring.meta[0], numpydoc.DocstringParam) + assert docstring.meta[0].arg_name == 'only_seldom_used_keywords' + assert docstring.meta[0].type_name == 'type' + assert docstring.meta[0].is_optional + assert docstring.meta[0].description == 'Explanation' + + assert docstring.meta[1].args == [ + 'other_param', + 'common_parameters_listed_above', + ] + + +def test_yields() -> None: + """Test parsing yields.""" + docstring = parse( + """ + Short description + Yields + ------ + int + description + """, + ) + assert len(docstring.meta) == 1 + assert isinstance(docstring.meta[0], numpydoc.DocstringReturns) + assert docstring.meta[0].args == ['yields'] + assert docstring.meta[0].type_name == 'int' + assert docstring.meta[0].description == 'description' + assert docstring.meta[0].return_name is None + assert docstring.meta[0].is_generator + + +def test_returns() -> None: + """Test parsing returns.""" + docstring = parse( + """ + Short description + """, + ) + assert docstring.returns is None + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 0 + + docstring = parse( + """ + Short description + Returns + ------- + type + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'type' + assert docstring.returns.description is None + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Short description + Returns + ------- + int + description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == 'description' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Returns + ------- + Optional[Mapping[str, List[int]]] + A description: with a colon + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'Optional[Mapping[str, List[int]]]' + assert docstring.returns.description == 'A description: with a colon' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Short description + Returns + ------- + int + description + with much text + + even some spacing + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == ( + 'description\nwith much text\n\neven some spacing' + ) + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Short description + Returns + ------- + a : int + description for a + b : str + description for b + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == ('description for a') + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 2 + assert docstring.many_returns[0].type_name == 'int' + assert docstring.many_returns[0].description == 'description for a' + assert docstring.many_returns[0].return_name == 'a' + assert docstring.many_returns[1].type_name == 'str' + assert docstring.many_returns[1].description == 'description for b' + assert docstring.many_returns[1].return_name == 'b' + + +def test_raises() -> None: + """Test parsing raises.""" + docstring = parse( + """ + Short description + """, + ) + assert len(docstring.raises) == 0 + + docstring = parse( + """ + Short description + Raises + ------ + ValueError + description + """, + ) + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name == 'ValueError' + assert docstring.raises[0].description == 'description' + + +def test_warns() -> None: + """Test parsing warns.""" + docstring = parse( + """ + Short description + Warns + ----- + UserWarning + description + """, + ) + assert len(docstring.meta) == 1 + assert isinstance(docstring.meta[0], numpydoc.DocstringRaises) + assert docstring.meta[0].type_name == 'UserWarning' + assert docstring.meta[0].description == 'description' + + +def test_simple_sections() -> None: + """Test parsing simple sections.""" + docstring = parse( + """ + Short description + + See Also + -------- + something : some thing you can also see + actually, anything can go in this section + + Warnings + -------- + Here be dragons + + Notes + ----- + None of this is real + + References + ---------- + Cite the relevant literature, e.g. [1]_. You may also cite these + references in the notes section above. + + .. [1] O. McNoleg, "The integration of GIS, remote sensing, + expert systems and adaptive co-kriging for environmental habitat + modelling of the Highland Haggis using object-oriented, fuzzy-logic + and neural-network techniques," Computers & Geosciences, vol. 22, + pp. 585-588, 1996. + """, + ) + assert len(docstring.meta) == 4 + assert docstring.meta[0].args == ['see_also'] + assert docstring.meta[0].description == ( + 'something : some thing you can also see\n' + 'actually, anything can go in this section' + ) + + assert docstring.meta[1].args == ['warnings'] + assert docstring.meta[1].description == 'Here be dragons' + + assert docstring.meta[2].args == ['notes'] + assert docstring.meta[2].description == 'None of this is real' + + assert docstring.meta[3].args == ['references'] + + +@pytest.mark.parametrize( + 'source, expected_results', + [ + ( + 'Description\nExamples\n--------\nlong example\n\nmore here', + [ + (None, 'long example\n\nmore here'), + ], + ), + ( + 'Description\nExamples\n--------\n>>> test', + [ + ('>>> test', ''), + ], + ), + ( + 'Description\nExamples\n--------\n>>> testa\n>>> testb', + [ + ('>>> testa\n>>> testb', ''), + ], + ), + ( + 'Description\nExamples\n--------\n>>> test1\ndesc1', + [ + ('>>> test1\ndesc1', ''), + ], + ), + ( + 'Description\nExamples\n--------\n' + '>>> test1a\n>>> test1b\ndesc1a\ndesc1b', + [ + ('>>> test1a\n>>> test1b\ndesc1a\ndesc1b', ''), + ], + ), + ( + 'Description\nExamples\n--------\n' + '>>> test1\ndesc1\n>>> test2\ndesc2', + [ + ('>>> test1\ndesc1', ''), + ('>>> test2\ndesc2', ''), + ], + ), + ( + 'Description\nExamples\n--------\n' + '>>> test1a\n>>> test1b\ndesc1a\ndesc1b\n' + '>>> test2a\n>>> test2b\ndesc2a\ndesc2b\n', + [ + ('>>> test1a\n>>> test1b\ndesc1a\ndesc1b', ''), + ('>>> test2a\n>>> test2b\ndesc2a\ndesc2b', ''), + ], + ), + ( + 'Description\nExamples\n--------\n' + ' >>> test1a\n >>> test1b\n desc1a\n desc1b\n' + ' >>> test2a\n >>> test2b\n desc2a\n desc2b\n', + [ + ('>>> test1a\n>>> test1b\ndesc1a\ndesc1b', ''), + ('>>> test2a\n>>> test2b\ndesc2a\ndesc2b', ''), + ], + ), + ], +) +def test_examples( + source: str, expected_results: T.List[T.Tuple[T.Optional[str], str]], +) -> None: + """Test parsing examples.""" + docstring = parse(source) + assert len(docstring.meta) == len(expected_results) + for meta, expected_result in zip(docstring.meta, expected_results): + assert meta.description == expected_result[1] + assert len(docstring.examples) == len(expected_results) + for example, expected_result in zip(docstring.examples, expected_results): + assert example.snippet == expected_result[0] + assert example.description == expected_result[1] + + +@pytest.mark.parametrize( + 'source, expected_depr_version, expected_depr_desc', + [ + ( + 'Short description\n\n.. deprecated:: 1.6.0\n This is busted!', + '1.6.0', + 'This is busted!', + ), + ( + ( + 'Short description\n\n' + '.. deprecated:: 1.6.0\n' + ' This description has\n' + ' multiple lines!' + ), + '1.6.0', + 'This description has\nmultiple lines!', + ), + ('Short description\n\n.. deprecated:: 1.6.0', '1.6.0', None), + ( + 'Short description\n\n.. deprecated::\n No version!', + None, + 'No version!', + ), + ], +) +def test_deprecation( + source: str, + expected_depr_version: T.Optional[str], + expected_depr_desc: T.Optional[str], +) -> None: + """Test parsing deprecation notes.""" + docstring = parse(source) + + assert docstring.deprecation is not None + assert docstring.deprecation.version == expected_depr_version + assert docstring.deprecation.description == expected_depr_desc + + +@pytest.mark.parametrize( + 'source, expected', + [ + ('', ''), + ('\n', ''), + ('Short description', 'Short description'), + ('\nShort description\n', 'Short description'), + ('\n Short description\n', 'Short description'), + ( + 'Short description\n\nLong description', + 'Short description\n\nLong description', + ), + ( + """ + Short description + + Long description + """, + 'Short description\n\nLong description', + ), + ( + """ + Short description + + Long description + Second line + """, + 'Short description\n\nLong description\nSecond line', + ), + ( + 'Short description\nLong description', + 'Short description\nLong description', + ), + ( + """ + Short description + Long description + """, + 'Short description\nLong description', + ), + ( + '\nShort description\nLong description\n', + 'Short description\nLong description', + ), + ( + """ + Short description + Long description + Second line + """, + 'Short description\nLong description\nSecond line', + ), + ( + """ + Short description + Meta: + ----- + asd + """, + 'Short description\nMeta:\n-----\n asd', + ), + ( + """ + Short description + Long description + Meta: + ----- + asd + """, + 'Short description\n' + 'Long description\n' + 'Meta:\n' + '-----\n' + ' asd', + ), + ( + """ + Short description + First line + Second line + Meta: + ----- + asd + """, + 'Short description\n' + 'First line\n' + ' Second line\n' + 'Meta:\n' + '-----\n' + ' asd', + ), + ( + """ + Short description + + First line + Second line + Meta: + ----- + asd + """, + 'Short description\n' + '\n' + 'First line\n' + ' Second line\n' + 'Meta:\n' + '-----\n' + ' asd', + ), + ( + """ + Short description + + First line + Second line + + Meta: + ----- + asd + """, + 'Short description\n' + '\n' + 'First line\n' + ' Second line\n' + '\n' + 'Meta:\n' + '-----\n' + ' asd', + ), + ( + """ + Short description + + Meta: + ----- + asd + 1 + 2 + 3 + """, + 'Short description\n' + '\n' + 'Meta:\n' + '-----\n' + ' asd\n' + ' 1\n' + ' 2\n' + ' 3', + ), + ( + """ + Short description + + Meta1: + ------ + asd + 1 + 2 + 3 + Meta2: + ------ + herp + Meta3: + ------ + derp + """, + 'Short description\n' + '\n' + 'Meta1:\n' + '------\n' + ' asd\n' + ' 1\n' + ' 2\n' + ' 3\n' + 'Meta2:\n' + '------\n' + ' herp\n' + 'Meta3:\n' + '------\n' + ' derp', + ), + ( + """ + Short description + + Parameters: + ----------- + name + description 1 + priority: int + description 2 + sender: str, optional + description 3 + message: str, optional + description 4, defaults to 'hello' + multiline: str, optional + long description 5, + defaults to 'bye' + """, + 'Short description\n' + '\n' + 'Parameters:\n' + '-----------\n' + ' name\n' + ' description 1\n' + ' priority: int\n' + ' description 2\n' + ' sender: str, optional\n' + ' description 3\n' + ' message: str, optional\n' + " description 4, defaults to 'hello'\n" + ' multiline: str, optional\n' + ' long description 5,\n' + " defaults to 'bye'", + ), + ( + """ + Short description + Raises: + ------- + ValueError + description + """, + 'Short description\n' + 'Raises:\n' + '-------\n' + ' ValueError\n' + ' description', + ), + ( + """ + Description + Examples: + -------- + >>> test1a + >>> test1b + desc1a + desc1b + >>> test2a + >>> test2b + desc2a + desc2b + """, + 'Description\n' + 'Examples:\n' + '--------\n' + '>>> test1a\n' + '>>> test1b\n' + 'desc1a\n' + 'desc1b\n' + '>>> test2a\n' + '>>> test2b\n' + 'desc2a\n' + 'desc2b', + ), + ], +) +def test_compose(source: str, expected: str) -> None: + """Test compose in default mode.""" + assert compose(parse(source)) == expected diff --git a/singlestoredb/docstring/tests/test_parse_from_object.py b/singlestoredb/docstring/tests/test_parse_from_object.py new file mode 100644 index 000000000..ed50718ea --- /dev/null +++ b/singlestoredb/docstring/tests/test_parse_from_object.py @@ -0,0 +1,109 @@ +"""Tests for parse_from_object function and attribute docstrings.""" +from unittest.mock import patch + +from singlestoredb.docstring import parse_from_object + +# module_attr: int = 1 +# """Description for module_attr""" + + +# def test_from_module_attribute_docstrings() -> None: +# """Test the parse of attribute docstrings from a module.""" +# from . import test_parse_from_object # pylint: disable=C0415,W0406 +# +# docstring = parse_from_object(test_parse_from_object) +# +# assert 'parse_from_object' in docstring.short_description +# assert len(docstring.params) == 1 +# assert docstring.params[0].arg_name == 'module_attr' +# assert docstring.params[0].type_name == 'int' +# assert docstring.params[0].description == 'Description for module_attr' + + +def test_from_class_attribute_docstrings() -> None: + """Test the parse of attribute docstrings from a class.""" + + class StandardCase: + """Short description + Long description + """ + + attr_one: str + """Description for attr_one""" + attr_two: bool = False + """Description for attr_two""" + + docstring = parse_from_object(StandardCase) + + assert docstring.short_description == 'Short description' + assert docstring.long_description == 'Long description' + assert docstring.description == 'Short description\nLong description' + assert len(docstring.params) == 2 + assert docstring.params[0].arg_name == 'attr_one' + assert docstring.params[0].type_name == 'str' + assert docstring.params[0].description == 'Description for attr_one' + assert docstring.params[1].arg_name == 'attr_two' + assert docstring.params[1].type_name == 'bool' + assert docstring.params[1].description == 'Description for attr_two' + + +def test_from_class_attribute_docstrings_without_type() -> None: + """Test the parse of untyped attribute docstrings.""" + + class WithoutType: # pylint: disable=missing-class-docstring + attr_one = 'value' + """Description for attr_one""" + + docstring = parse_from_object(WithoutType) + + assert docstring.short_description is None + assert docstring.long_description is None + assert docstring.description is None + assert len(docstring.params) == 1 + assert docstring.params[0].arg_name == 'attr_one' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'Description for attr_one' + + +def test_from_class_without_source() -> None: + """Test the parse of class when source is unavailable.""" + + class WithoutSource: + """Short description""" + + attr_one: str + """Description for attr_one""" + + with patch( + 'inspect.getsource', side_effect=OSError('could not get source code'), + ): + docstring = parse_from_object(WithoutSource) + + assert docstring.short_description == 'Short description' + assert docstring.long_description is None + assert docstring.description == 'Short description' + assert len(docstring.params) == 0 + + +def test_from_function() -> None: + """Test the parse of a function docstring.""" + + def a_function(param1: str, param2: int = 2) -> str: + """Short description + Args: + param1: Description for param1 + param2: Description for param2 + """ + return f'{param1} {param2}' + + docstring = parse_from_object(a_function) + + assert docstring.short_description == 'Short description' + assert docstring.description == 'Short description' + assert len(docstring.params) == 2 + assert docstring.params[0].arg_name == 'param1' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'Description for param1' + assert docstring.params[1].arg_name == 'param2' + assert docstring.params[1].type_name is None + assert docstring.params[1].description == 'Description for param2' diff --git a/singlestoredb/docstring/tests/test_parser.py b/singlestoredb/docstring/tests/test_parser.py new file mode 100644 index 000000000..fcecbb783 --- /dev/null +++ b/singlestoredb/docstring/tests/test_parser.py @@ -0,0 +1,248 @@ +"""Tests for generic docstring routines.""" +import typing as T + +import pytest + +from singlestoredb.docstring.common import DocstringStyle +from singlestoredb.docstring.common import ParseError +from singlestoredb.docstring.parser import parse + + +@pytest.mark.parametrize( + 'source, expected', + [ + pytest.param(None, None, id='No __doc__'), + ('', None), + ('\n', None), + ('Short description', 'Short description'), + ('\nShort description\n', 'Short description'), + ('\n Short description\n', 'Short description'), + ], +) +def test_short_description( + source: T.Optional[str], expected: T.Optional[str], +) -> None: + """Test parsing short description.""" + docstring = parse(source) + assert docstring.short_description == expected + assert docstring.description == expected + assert docstring.long_description is None + assert not docstring.meta + + +def test_rest() -> None: + """Test ReST-style parser autodetection.""" + docstring = parse( + """ + Short description + + Long description + + Causing people to indent: + + A lot sometimes + + :param spam: spam desc + :param int bla: bla desc + :param str yay: + :raises ValueError: exc desc + :returns tuple: ret desc + """, + ) + + assert docstring.style == DocstringStyle.REST + assert docstring.short_description == 'Short description' + assert docstring.long_description == ( + 'Long description\n\n' + 'Causing people to indent:\n\n' + ' A lot sometimes' + ) + assert docstring.description == ( + 'Short description\n\n' + 'Long description\n\n' + 'Causing people to indent:\n\n' + ' A lot sometimes' + ) + assert len(docstring.params) == 3 + assert docstring.params[0].arg_name == 'spam' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'spam desc' + assert docstring.params[1].arg_name == 'bla' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'bla desc' + assert docstring.params[2].arg_name == 'yay' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description == '' + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name == 'ValueError' + assert docstring.raises[0].description == 'exc desc' + assert docstring.returns is not None + assert docstring.returns.type_name == 'tuple' + assert docstring.returns.description == 'ret desc' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + +def test_google() -> None: + """Test Google-style parser autodetection.""" + docstring = parse( + """Short description + + Long description + + Causing people to indent: + + A lot sometimes + + Args: + spam: spam desc + bla (int): bla desc + yay (str): + + Raises: + ValueError: exc desc + + Returns: + tuple: ret desc + """, + ) + + assert docstring.style == DocstringStyle.GOOGLE + assert docstring.short_description == 'Short description' + assert docstring.long_description == ( + 'Long description\n\n' + 'Causing people to indent:\n\n' + ' A lot sometimes' + ) + assert docstring.description == ( + 'Short description\n\n' + 'Long description\n\n' + 'Causing people to indent:\n\n' + ' A lot sometimes' + ) + assert len(docstring.params) == 3 + assert docstring.params[0].arg_name == 'spam' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'spam desc' + assert docstring.params[1].arg_name == 'bla' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'bla desc' + assert docstring.params[2].arg_name == 'yay' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description == '' + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name == 'ValueError' + assert docstring.raises[0].description == 'exc desc' + assert docstring.returns is not None + assert docstring.returns.type_name == 'tuple' + assert docstring.returns.description == 'ret desc' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + +def test_numpydoc() -> None: + """Test numpydoc-style parser autodetection.""" + docstring = parse( + """Short description + + Long description + + Causing people to indent: + + A lot sometimes + + Parameters + ---------- + spam + spam desc + bla : int + bla desc + yay : str + + Raises + ------ + ValueError + exc desc + + Other Parameters + ---------------- + this_guy : int, optional + you know him + + Returns + ------- + tuple + ret desc + + See Also + -------- + multiple lines... + something else? + + Warnings + -------- + multiple lines... + none of this is real! + """, + ) + + assert docstring.style == DocstringStyle.NUMPYDOC + assert docstring.short_description == 'Short description' + assert docstring.long_description == ( + 'Long description\n\n' + 'Causing people to indent:\n\n' + ' A lot sometimes' + ) + assert docstring.description == ( + 'Short description\n\n' + 'Long description\n\n' + 'Causing people to indent:\n\n' + ' A lot sometimes' + ) + assert len(docstring.params) == 4 + assert docstring.params[0].arg_name == 'spam' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'spam desc' + assert docstring.params[1].arg_name == 'bla' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'bla desc' + assert docstring.params[2].arg_name == 'yay' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description is None + assert docstring.params[3].arg_name == 'this_guy' + assert docstring.params[3].type_name == 'int' + assert docstring.params[3].is_optional + assert docstring.params[3].description == 'you know him' + + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name == 'ValueError' + assert docstring.raises[0].description == 'exc desc' + assert docstring.returns is not None + assert docstring.returns.type_name == 'tuple' + assert docstring.returns.description == 'ret desc' + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + +def test_autodetection_error_detection() -> None: + """Test autodection for the case where one of the parsers throws an error + and another one succeeds. + """ + source = """ + Does something useless + + :param 3 + 3 a: a param + """ + + with pytest.raises(ParseError): + # assert that one of the parsers does raise + parse(source, DocstringStyle.REST) + + # assert that autodetection still works + docstring = parse(source) + + assert docstring + assert docstring.style == DocstringStyle.GOOGLE diff --git a/singlestoredb/docstring/tests/test_rest.py b/singlestoredb/docstring/tests/test_rest.py new file mode 100644 index 000000000..949db8083 --- /dev/null +++ b/singlestoredb/docstring/tests/test_rest.py @@ -0,0 +1,547 @@ +"""Tests for ReST-style docstring routines.""" +import typing as T + +import pytest + +from singlestoredb.docstring.common import ParseError +from singlestoredb.docstring.common import RenderingStyle +from singlestoredb.docstring.rest import compose +from singlestoredb.docstring.rest import parse + + +@pytest.mark.parametrize( + 'source, expected', + [ + pytest.param(None, None, id='No __doc__'), + ('', None), + ('\n', None), + ('Short description', 'Short description'), + ('\nShort description\n', 'Short description'), + ('\n Short description\n', 'Short description'), + ], +) +def test_short_description( + source: T.Optional[str], expected: T.Optional[str], +) -> None: + """Test parsing short description.""" + docstring = parse(source) + assert docstring.short_description == expected + assert docstring.description == expected + assert docstring.long_description is None + assert not docstring.meta + + +@pytest.mark.parametrize( + 'source, expected_short_desc, expected_long_desc, expected_blank', + [ + ( + 'Short description\n\nLong description', + 'Short description', + 'Long description', + True, + ), + ( + """ + Short description + + Long description + """, + 'Short description', + 'Long description', + True, + ), + ( + """ + Short description + + Long description + Second line + """, + 'Short description', + 'Long description\nSecond line', + True, + ), + ( + 'Short description\nLong description', + 'Short description', + 'Long description', + False, + ), + ( + """ + Short description + Long description + """, + 'Short description', + 'Long description', + False, + ), + ( + '\nShort description\nLong description\n', + 'Short description', + 'Long description', + False, + ), + ( + """ + Short description + Long description + Second line + """, + 'Short description', + 'Long description\nSecond line', + False, + ), + ], +) +def test_long_description( + source: str, + expected_short_desc: str, + expected_long_desc: str, + expected_blank: bool, +) -> None: + """Test parsing long description.""" + docstring = parse(source) + assert docstring.short_description == expected_short_desc + assert docstring.long_description == expected_long_desc + assert docstring.blank_after_short_description == expected_blank + assert not docstring.meta + + +@pytest.mark.parametrize( + 'source, expected_short_desc, expected_long_desc, ' + 'expected_blank_short_desc, expected_blank_long_desc, ' + 'expected_full_desc', + [ + ( + """ + Short description + :meta: asd + """, + 'Short description', + None, + False, + False, + 'Short description', + ), + ( + """ + Short description + Long description + :meta: asd + """, + 'Short description', + 'Long description', + False, + False, + 'Short description\nLong description', + ), + ( + """ + Short description + First line + Second line + :meta: asd + """, + 'Short description', + 'First line\n Second line', + False, + False, + 'Short description\nFirst line\n Second line', + ), + ( + """ + Short description + + First line + Second line + :meta: asd + """, + 'Short description', + 'First line\n Second line', + True, + False, + 'Short description\n\nFirst line\n Second line', + ), + ( + """ + Short description + + First line + Second line + + :meta: asd + """, + 'Short description', + 'First line\n Second line', + True, + True, + 'Short description\n\nFirst line\n Second line', + ), + ( + """ + :meta: asd + """, + None, + None, + False, + False, + None, + ), + ], +) +def test_meta_newlines( + source: str, + expected_short_desc: T.Optional[str], + expected_long_desc: T.Optional[str], + expected_blank_short_desc: bool, + expected_blank_long_desc: bool, + expected_full_desc: T.Optional[str], +) -> None: + """Test parsing newlines around description sections.""" + docstring = parse(source) + assert docstring.short_description == expected_short_desc + assert docstring.long_description == expected_long_desc + assert docstring.blank_after_short_description == expected_blank_short_desc + assert docstring.blank_after_long_description == expected_blank_long_desc + assert docstring.description == expected_full_desc + assert len(docstring.meta) == 1 + + +def test_meta_with_multiline_description() -> None: + """Test parsing multiline meta documentation.""" + docstring = parse( + """ + Short description + + :meta: asd + 1 + 2 + 3 + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 1 + assert docstring.meta[0].args == ['meta'] + assert docstring.meta[0].description == 'asd\n1\n 2\n3' + + +def test_multiple_meta() -> None: + """Test parsing multiple meta.""" + docstring = parse( + """ + Short description + + :meta1: asd + 1 + 2 + 3 + :meta2: herp + :meta3: derp + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 3 + assert docstring.meta[0].args == ['meta1'] + assert docstring.meta[0].description == 'asd\n1\n 2\n3' + assert docstring.meta[1].args == ['meta2'] + assert docstring.meta[1].description == 'herp' + assert docstring.meta[2].args == ['meta3'] + assert docstring.meta[2].description == 'derp' + + +def test_meta_with_args() -> None: + """Test parsing meta with additional arguments.""" + docstring = parse( + """ + Short description + + :meta ene due rabe: asd + """, + ) + assert docstring.short_description == 'Short description' + assert len(docstring.meta) == 1 + assert docstring.meta[0].args == ['meta', 'ene', 'due', 'rabe'] + assert docstring.meta[0].description == 'asd' + + +def test_params() -> None: + """Test parsing params.""" + docstring = parse('Short description') + assert len(docstring.params) == 0 + + docstring = parse( + """ + Short description + + :param name: description 1 + :param int priority: description 2 + :param str? sender: description 3 + :param str? message: description 4, defaults to 'hello' + :param str? multiline: long description 5, + defaults to 'bye' + """, + ) + assert len(docstring.params) == 5 + assert docstring.params[0].arg_name == 'name' + assert docstring.params[0].type_name is None + assert docstring.params[0].description == 'description 1' + assert docstring.params[0].default is None + assert not docstring.params[0].is_optional + assert docstring.params[1].arg_name == 'priority' + assert docstring.params[1].type_name == 'int' + assert docstring.params[1].description == 'description 2' + assert not docstring.params[1].is_optional + assert docstring.params[1].default is None + assert docstring.params[2].arg_name == 'sender' + assert docstring.params[2].type_name == 'str' + assert docstring.params[2].description == 'description 3' + assert docstring.params[2].is_optional + assert docstring.params[2].default is None + assert docstring.params[3].arg_name == 'message' + assert docstring.params[3].type_name == 'str' + assert ( + docstring.params[3].description == "description 4, defaults to 'hello'" + ) + assert docstring.params[3].is_optional + assert docstring.params[3].default == "'hello'" + assert docstring.params[4].arg_name == 'multiline' + assert docstring.params[4].type_name == 'str' + assert ( + docstring.params[4].description + == "long description 5,\ndefaults to 'bye'" + ) + assert docstring.params[4].is_optional + assert docstring.params[4].default == "'bye'" + + docstring = parse( + """ + Short description + + :param a: description a + :type a: int + :param int b: description b + """, + ) + assert len(docstring.params) == 2 + assert docstring.params[0].arg_name == 'a' + assert docstring.params[0].type_name == 'int' + assert docstring.params[0].description == 'description a' + assert docstring.params[0].default is None + assert not docstring.params[0].is_optional + + +def test_returns() -> None: + """Test parsing returns.""" + docstring = parse( + """ + Short description + """, + ) + assert docstring.returns is None + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 0 + + docstring = parse( + """ + Short description + :returns: description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name is None + assert docstring.returns.description == 'description' + assert not docstring.returns.is_generator + assert docstring.many_returns == [docstring.returns] + + docstring = parse( + """ + Short description + :returns int: description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == 'description' + assert not docstring.returns.is_generator + assert docstring.many_returns == [docstring.returns] + + docstring = parse( + """ + Short description + :returns: description + :rtype: int + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == 'description' + assert not docstring.returns.is_generator + assert docstring.many_returns == [docstring.returns] + + +def test_yields() -> None: + """Test parsing yields.""" + docstring = parse( + """ + Short description + """, + ) + assert docstring.returns is None + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 0 + + docstring = parse( + """ + Short description + :yields: description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name is None + assert docstring.returns.description == 'description' + assert docstring.returns.is_generator + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + docstring = parse( + """ + Short description + :yields int: description + """, + ) + assert docstring.returns is not None + assert docstring.returns.type_name == 'int' + assert docstring.returns.description == 'description' + assert docstring.returns.is_generator + assert docstring.many_returns is not None + assert len(docstring.many_returns) == 1 + assert docstring.many_returns[0] == docstring.returns + + +def test_raises() -> None: + """Test parsing raises.""" + docstring = parse( + """ + Short description + """, + ) + assert len(docstring.raises) == 0 + + docstring = parse( + """ + Short description + :raises: description + """, + ) + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name is None + assert docstring.raises[0].description == 'description' + + docstring = parse( + """ + Short description + :raises ValueError: description + """, + ) + assert len(docstring.raises) == 1 + assert docstring.raises[0].type_name == 'ValueError' + assert docstring.raises[0].description == 'description' + + +def test_broken_meta() -> None: + """Test parsing broken meta.""" + with pytest.raises(ParseError): + parse(':') + + with pytest.raises(ParseError): + parse(':param herp derp') + + with pytest.raises(ParseError): + parse(':param: invalid') + + with pytest.raises(ParseError): + parse(':param with too many args: desc') + + # these should not raise any errors + parse(':sthstrange: desc') + + +def test_deprecation() -> None: + """Test parsing deprecation notes.""" + docstring = parse(':deprecation: 1.1.0 this function will be removed') + assert docstring.deprecation is not None + assert docstring.deprecation.version == '1.1.0' + assert docstring.deprecation.description == 'this function will be removed' + + docstring = parse(':deprecation: this function will be removed') + assert docstring.deprecation is not None + assert docstring.deprecation.version is None + assert docstring.deprecation.description == 'this function will be removed' + + +@pytest.mark.parametrize( + 'rendering_style, expected', + [ + ( + RenderingStyle.COMPACT, + 'Short description.\n' + '\n' + 'Long description.\n' + '\n' + ':param int foo: a description\n' + ':param int bar: another description\n' + ':returns float: a return', + ), + ( + RenderingStyle.CLEAN, + 'Short description.\n' + '\n' + 'Long description.\n' + '\n' + ':param int foo: a description\n' + ':param int bar: another description\n' + ':returns float: a return', + ), + ( + RenderingStyle.EXPANDED, + 'Short description.\n' + '\n' + 'Long description.\n' + '\n' + ':param foo:\n' + ' a description\n' + ':type foo: int\n' + ':param bar:\n' + ' another description\n' + ':type bar: int\n' + ':returns:\n' + ' a return\n' + ':rtype: float', + ), + ], +) +def test_compose(rendering_style: RenderingStyle, expected: str) -> None: + """Test compose""" + + docstring = parse( + """ + Short description. + + Long description. + + :param int foo: a description + :param int bar: another description + :return float: a return + """, + ) + assert compose(docstring, rendering_style=rendering_style) == expected + + +def test_short_rtype() -> None: + """Test abbreviated docstring with only return type information.""" + string = 'Short description.\n\n:rtype: float' + docstring = parse(string) + rendering_style = RenderingStyle.EXPANDED + assert compose(docstring, rendering_style=rendering_style) == string diff --git a/singlestoredb/docstring/tests/test_util.py b/singlestoredb/docstring/tests/test_util.py new file mode 100644 index 000000000..ceaf10dd4 --- /dev/null +++ b/singlestoredb/docstring/tests/test_util.py @@ -0,0 +1,70 @@ +"""Test for utility functions.""" +from typing import Any + +from singlestoredb.docstring.common import DocstringReturns +from singlestoredb.docstring.util import combine_docstrings + + +def test_combine_docstrings() -> None: + """Test combine_docstrings wrapper.""" + + def fun1(arg_a: Any, arg_b: Any, arg_c: Any, arg_d: Any) -> None: + """short_description: fun1 + + :param arg_a: fun1 + :param arg_b: fun1 + :return: fun1 + """ + assert arg_a and arg_b and arg_c and arg_d + + def fun2(arg_b: Any, arg_c: Any, arg_d: Any, arg_e: Any) -> None: + """short_description: fun2 + + long_description: fun2 + + :param arg_b: fun2 + :param arg_c: fun2 + :param arg_e: fun2 + """ + assert arg_b and arg_c and arg_d and arg_e + + @combine_docstrings(fun1, fun2) + def decorated1( + arg_a: Any, arg_b: Any, arg_c: Any, + arg_d: Any, arg_e: Any, arg_f: Any, + ) -> None: + """ + :param arg_e: decorated + :param arg_f: decorated + """ + assert arg_a and arg_b and arg_c and arg_d and arg_e and arg_f + + assert decorated1.__doc__ == ( + 'short_description: fun2\n' + '\n' + 'long_description: fun2\n' + '\n' + ':param arg_a: fun1\n' + ':param arg_b: fun1\n' + ':param arg_c: fun2\n' + ':param arg_e: fun2\n' + ':param arg_f: decorated\n' + ':returns: fun1' + ) + + @combine_docstrings(fun1, fun2, exclude=[DocstringReturns]) + def decorated2( + arg_a: Any, arg_b: Any, arg_c: Any, arg_d: Any, arg_e: Any, arg_f: Any, + ) -> None: + assert arg_a and arg_b and arg_c and arg_d and arg_e and arg_f + + assert decorated2.__doc__ == ( + 'short_description: fun2\n' + '\n' + 'long_description: fun2\n' + '\n' + ':param arg_a: fun1\n' + ':param arg_b: fun1\n' + ':param arg_c: fun2\n' + ':param arg_e: fun2' + ) diff --git a/singlestoredb/docstring/util.py b/singlestoredb/docstring/util.py new file mode 100644 index 000000000..0b891e1cf --- /dev/null +++ b/singlestoredb/docstring/util.py @@ -0,0 +1,141 @@ +"""Utility functions for working with docstrings.""" +import typing as T +from collections import ChainMap +from inspect import Signature +from itertools import chain + +from .common import DocstringMeta +from .common import DocstringParam +from .common import DocstringReturns # noqa: F401 +from .common import DocstringStyle +from .common import RenderingStyle +from .parser import compose +from .parser import parse + +_Func = T.Callable[..., T.Any] + + +def combine_docstrings( + *others: _Func, + exclude: T.Iterable[T.Type[DocstringMeta]] = (), + style: DocstringStyle = DocstringStyle.AUTO, + rendering_style: RenderingStyle = RenderingStyle.COMPACT, +) -> _Func: + """A function decorator that parses the docstrings from `others`, + programmatically combines them with the parsed docstring of the decorated + function, and replaces the docstring of the decorated function with the + composed result. Only parameters that are part of the decorated functions + signature are included in the combined docstring. When multiple sources for + a parameter or docstring metadata exists then the decorator will first + default to the wrapped function's value (when available) and otherwise use + the rightmost definition from ``others``. + + The following example illustrates its usage: + + >>> def fun1(a, b, c, d): + ... '''short_description: fun1 + ... + ... :param a: fun1 + ... :param b: fun1 + ... :return: fun1 + ... ''' + >>> def fun2(b, c, d, e): + ... '''short_description: fun2 + ... + ... long_description: fun2 + ... + ... :param b: fun2 + ... :param c: fun2 + ... :param e: fun2 + ... ''' + >>> @combine_docstrings(fun1, fun2) + >>> def decorated(a, b, c, d, e, f): + ... ''' + ... :param e: decorated + ... :param f: decorated + ... ''' + >>> print(decorated.__doc__) + short_description: fun2 + + long_description: fun2 + + :param a: fun1 + :param b: fun1 + :param c: fun2 + :param e: fun2 + :param f: decorated + :returns: fun1 + >>> @combine_docstrings(fun1, fun2, exclude=[DocstringReturns]) + >>> def decorated(a, b, c, d, e, f): pass + >>> print(decorated.__doc__) + short_description: fun2 + + long_description: fun2 + + :param a: fun1 + :param b: fun1 + :param c: fun2 + :param e: fun2 + + :param others: callables from which to parse docstrings. + :param exclude: an iterable of ``DocstringMeta`` subclasses to exclude when + combining docstrings. + :param style: style composed docstring. The default will infer the style + from the decorated function. + :param rendering_style: The rendering style used to compose a docstring. + :return: the decorated function with a modified docstring. + """ + + def wrapper(func: _Func) -> _Func: + sig = Signature.from_callable(func) + + comb_doc = parse(func.__doc__ or '') + docs = [parse(other.__doc__ or '') for other in others] + [comb_doc] + params = dict( + ChainMap( + *( + {param.arg_name: param for param in doc.params} + for doc in docs + ), + ), + ) + + for doc in reversed(docs): + if not doc.short_description: + continue + comb_doc.short_description = doc.short_description + comb_doc.blank_after_short_description = ( + doc.blank_after_short_description + ) + break + + for doc in reversed(docs): + if not doc.long_description: + continue + comb_doc.long_description = doc.long_description + comb_doc.blank_after_long_description = ( + doc.blank_after_long_description + ) + break + + combined: T.Dict[T.Type[DocstringMeta], T.List[DocstringMeta]] = {} + for doc in docs: + metas: T.Dict[T.Type[DocstringMeta], T.List[DocstringMeta]] = {} + for meta in doc.meta: + meta_type = type(meta) + if meta_type in exclude: + continue + metas.setdefault(meta_type, []).append(meta) + for meta_type, meta_list in metas.items(): + combined[meta_type] = meta_list + + combined[DocstringParam] = [ + params[name] for name in sig.parameters if name in params + ] + comb_doc.meta = list(chain(*combined.values())) + func.__doc__ = compose( + comb_doc, style=style, rendering_style=rendering_style, + ) + return func + + return wrapper diff --git a/singlestoredb/functions/ext/asgi.py b/singlestoredb/functions/ext/asgi.py index 69b498bd4..578df3e8c 100755 --- a/singlestoredb/functions/ext/asgi.py +++ b/singlestoredb/functions/ext/asgi.py @@ -73,6 +73,8 @@ from ..typing import Masked from ..typing import Table from .timer import Timer +from singlestoredb.docstring.parser import parse +from singlestoredb.functions.dtypes import escape_name try: import cloudpickle @@ -538,6 +540,8 @@ def make_func( Name of the function to create func : Callable The function to call as the endpoint + database : str, optional + The database to use for the function definition Returns ------- @@ -615,7 +619,7 @@ async def cancel_on_disconnect( """Cancel request if client disconnects.""" while True: message = await receive() - if message['type'] == 'http.disconnect': + if message.get('type', '') == 'http.disconnect': raise asyncio.CancelledError( 'Function call was cancelled by client', ) @@ -674,6 +678,8 @@ class Application(object): link_credentials : Dict[str, Any], optional The CREDENTIALS section of a LINK definition. This dictionary gets converted to JSON for the CREATE LINK call. + function_database : str, optional + The database to use for external function definitions. """ @@ -816,6 +822,7 @@ class Application(object): invoke_path = ('invoke',) show_create_function_path = ('show', 'create_function') show_function_info_path = ('show', 'function_info') + status = ('status',) def __init__( self, @@ -838,6 +845,7 @@ def __init__( link_credentials: Optional[Dict[str, Any]] = None, name_prefix: str = get_option('external_function.name_prefix'), name_suffix: str = get_option('external_function.name_suffix'), + function_database: Optional[str] = None, ) -> None: if link_name and (link_config or link_credentials): raise ValueError( @@ -944,6 +952,7 @@ def __init__( self.link_credentials = link_credentials self.endpoints = endpoints self.external_functions = external_functions + self.function_database = function_database async def __call__( self, @@ -992,6 +1001,7 @@ async def __call__( accepts = headers.get(b'accepts', content_type) func_name = headers.get(b's2-ef-name', b'') func_endpoint = self.endpoints.get(func_name) + ignore_cancel = headers.get(b's2-ef-ignore-cancel', b'false') == b'true' timer.metadata['function'] = func_name.decode('utf-8') if func_name else '' call_timer.metadata['function'] = timer.metadata['function'] @@ -1021,7 +1031,7 @@ async def __call__( with timer('receive_data'): while more_body: request = await receive() - if request['type'] == 'http.disconnect': + if request.get('type', '') == 'http.disconnect': raise RuntimeError('client disconnected') data.append(request['body']) more_body = request.get('more_body', False) @@ -1051,7 +1061,8 @@ async def __call__( ), ) disconnect_task = asyncio.create_task( - cancel_on_disconnect(receive), + asyncio.sleep(int(1e9)) + if ignore_cancel else cancel_on_disconnect(receive), ) timeout_task = asyncio.create_task( cancel_on_timeout(func_info['timeout']), @@ -1130,6 +1141,7 @@ async def __call__( endpoint_info['signature'], url=self.url or reflected_url, data_format=self.data_format, + database=self.function_database or None, ), ) body = '\n'.join(syntax).encode('utf-8') @@ -1142,6 +1154,11 @@ async def __call__( body = json.dumps(dict(functions=functions)).encode('utf-8') await send(self.text_response_dict) + # Return status + elif method == 'GET' and path == self.status: + body = json.dumps(dict(status='ok')).encode('utf-8') + await send(self.text_response_dict) + # Path not found else: body = b'' @@ -1184,20 +1201,27 @@ def _create_link( def _locate_app_functions(self, cur: Any) -> Tuple[Set[str], Set[str]]: """Locate all current functions and links belonging to this app.""" funcs, links = set(), set() - cur.execute('SHOW FUNCTIONS') + if self.function_database: + database_prefix = escape_name(self.function_database) + '.' + cur.execute(f'SHOW FUNCTIONS IN {escape_name(self.function_database)}') + else: + database_prefix = '' + cur.execute('SHOW FUNCTIONS') + for row in list(cur): name, ftype, link = row[0], row[1], row[-1] # Only look at external functions if 'external' not in ftype.lower(): continue # See if function URL matches url - cur.execute(f'SHOW CREATE FUNCTION `{name}`') + cur.execute(f'SHOW CREATE FUNCTION {database_prefix}{escape_name(name)}') for fname, _, code, *_ in list(cur): m = re.search(r" (?:\w+) (?:SERVICE|MANAGED) '([^']+)'", code) if m and m.group(1) == self.url: - funcs.add(fname) + funcs.add(f'{database_prefix}{escape_name(fname)}') if link and re.match(r'^py_ext_func_link_\S{14}$', link): links.add(link) + return funcs, links def get_function_info( @@ -1220,20 +1244,54 @@ def get_function_info( sig = info['signature'] sql_map[sig['name']] = sql - for key, (_, info) in self.endpoints.items(): + for key, (func, info) in self.endpoints.items(): + # Get info from docstring + doc_summary = '' + doc_long_description = '' + doc_params = {} + doc_returns = None + doc_examples = [] + if func.__doc__: + try: + docs = parse(func.__doc__) + doc_params = {p.arg_name: p for p in docs.params} + doc_returns = docs.returns + if not docs.short_description and docs.long_description: + doc_summary = docs.long_description or '' + else: + doc_summary = docs.short_description or '' + doc_long_description = docs.long_description or '' + for ex in docs.examples: + out = [] + if ex.description: + out.append(ex.description) + if ex.snippet: + out.append(ex.snippet) + if ex.post_snippet: + out.append(ex.post_snippet) + doc_examples.append('\n'.join(out)) + + except Exception as e: + logger.warning( + f'Could not parse docstring for function {key}: {e}', + ) + if not func_name or key == func_name: sig = info['signature'] args = [] # Function arguments - for a in sig.get('args', []): + for i, a in enumerate(sig.get('args', [])): + name = a['name'] dtype = a['dtype'] nullable = '?' in dtype args.append( dict( - name=a['name'], + name=name, dtype=dtype.replace('?', ''), nullable=nullable, + description=(doc_params[name].description or '') + if name in doc_params else '', ), ) if a.get('default', no_default) is not no_default: @@ -1250,6 +1308,8 @@ def get_function_info( dict( dtype=dtype.replace('?', ''), nullable=nullable, + description=doc_returns.description + if doc_returns else '', ), ) if a.get('name', None): @@ -1263,6 +1323,9 @@ def get_function_info( returns=returns, function_type=info['function_type'], sql_statement=sql, + summary=doc_summary, + long_description=doc_long_description, + examples=doc_examples, ) return functions @@ -1303,6 +1366,7 @@ def get_create_functions( app_mode=self.app_mode, replace=replace, link=link or None, + database=self.function_database or None, ), ) @@ -1332,7 +1396,7 @@ def register_functions( if replace: funcs, links = self._locate_app_functions(cur) for fname in funcs: - cur.execute(f'DROP FUNCTION IF EXISTS `{fname}`') + cur.execute(f'DROP FUNCTION IF EXISTS {fname}') for link in links: cur.execute(f'DROP LINK {link}') for func in self.get_create_functions(replace=replace): @@ -1358,7 +1422,7 @@ def drop_functions( with conn.cursor() as cur: funcs, links = self._locate_app_functions(cur) for fname in funcs: - cur.execute(f'DROP FUNCTION IF EXISTS `{fname}`') + cur.execute(f'DROP FUNCTION IF EXISTS {fname}') for link in links: cur.execute(f'DROP LINK {link}') @@ -1415,6 +1479,7 @@ async def send(content: Dict[str, Any]) -> None: b'accepts': accepts[data_format.lower()], b's2-ef-name': name.encode('utf-8'), b's2-ef-version': data_version.encode('utf-8'), + b's2-ef-ignore-cancel': b'true', }, ) @@ -1679,6 +1744,14 @@ def main(argv: Optional[List[str]] = None) -> None: ), help='Suffix to add to function names', ) + parser.add_argument( + '--function-database', metavar='function_database', + default=defaults.get( + 'function_database', + get_option('external_function.function_database'), + ), + help='Database to use for the function definition', + ) parser.add_argument( 'functions', metavar='module.or.func.path', nargs='*', help='functions or modules to export in UDF server', @@ -1778,6 +1851,7 @@ def main(argv: Optional[List[str]] = None) -> None: app_mode='remote', name_prefix=args.name_prefix, name_suffix=args.name_suffix, + function_database=args.function_database or None, ) funcs = app.get_create_functions(replace=args.replace_existing) diff --git a/singlestoredb/functions/signature.py b/singlestoredb/functions/signature.py index 35504401d..c9028d60a 100644 --- a/singlestoredb/functions/signature.py +++ b/singlestoredb/functions/signature.py @@ -1203,7 +1203,8 @@ def get_signature( f'{", ".join(args_data_formats)}', ) - out['args_data_format'] = args_data_formats[0] if args_data_formats else 'scalar' + adf = out['args_data_format'] = args_data_formats[0] \ + if args_data_formats else 'scalar' # Generate the return types and the corresponding SQL code for those values ret_schema, out['returns_data_format'], function_type = get_schema( @@ -1212,9 +1213,19 @@ def get_signature( mode='return', ) - out['returns_data_format'] = out['returns_data_format'] or 'scalar' + rdf = out['returns_data_format'] = out['returns_data_format'] or 'scalar' out['function_type'] = function_type + # Reality check the input and output data formats + if function_type == 'udf': + if (adf == 'scalar' and rdf != 'scalar') or \ + (adf != 'scalar' and rdf == 'scalar'): + raise TypeError( + 'Function can not have scalar arguments and a vector return type, ' + 'or vice versa. Parameters and return values must all be either ', + 'scalar or vector types.', + ) + # All functions have to return a value, so if none was specified try to # insert a reasonable default that includes NULLs. if not ret_schema: @@ -1370,6 +1381,7 @@ def signature_to_sql( app_mode: str = 'remote', link: Optional[str] = None, replace: bool = False, + database: Optional[str] = None, ) -> str: ''' Convert a dictionary function signature into SQL. @@ -1424,9 +1436,11 @@ def signature_to_sql( elif url is None: raise ValueError('url can not be `None`') - database = '' + database_prefix = '' if signature.get('database'): - database = escape_name(signature['database']) + '.' + database_prefix = escape_name(signature['database']) + '.' + elif database is not None: + database_prefix = escape_name(database) + '.' or_replace = 'OR REPLACE ' if (bool(signature.get('replace')) or replace) else '' @@ -1438,7 +1452,7 @@ def signature_to_sql( return ( f'CREATE {or_replace}EXTERNAL FUNCTION ' + - f'{database}{escape_name(signature["name"])}' + + f'{database_prefix}{escape_name(signature["name"])}' + '(' + ', '.join(args) + ')' + returns + f' AS {app_mode.upper()} SERVICE "{url}" FORMAT {data_format.upper()}' f'{link_str};' diff --git a/singlestoredb/tests/test_udf.py b/singlestoredb/tests/test_udf.py index 16eb325d6..e4b1217ef 100755 --- a/singlestoredb/tests/test_udf.py +++ b/singlestoredb/tests/test_udf.py @@ -36,6 +36,32 @@ def to_sql(x): class TestUDF(unittest.TestCase): + def test_invalid_signature(self): + + def foo(x: np.ndarray, y: np.ndarray) -> str: ... + with self.assertRaises(TypeError): + to_sql(foo) + + def foo(x: str, y: str) -> np.ndarray: ... + with self.assertRaises(TypeError): + to_sql(foo) + + def foo(x: str, y: np.ndarray) -> np.ndarray: ... + with self.assertRaises(TypeError): + to_sql(foo) + + def foo(x: np.ndarray, y: str) -> np.ndarray: ... + with self.assertRaises(TypeError): + to_sql(foo) + + def foo(x: str, y: np.ndarray) -> str: ... + with self.assertRaises(TypeError): + to_sql(foo) + + def foo(x: np.ndarray, y: str) -> str: ... + with self.assertRaises(TypeError): + to_sql(foo) + def test_return_annotations(self): # No annotations