Add function_database option; add docstring parsing for parameter des…

docs/src/api.rst

@@ -319,7 +319,7 @@ Stage Files
 ...........
 
 To interact with files in your Stage, use the
-:attr:`WorkspaceManager.stage` attribute.
+:attr:`WorkspaceGroup.stage` attribute.
 It will return a :class:`Stage` object which defines the following
 methods and attributes.
 

singlestoredb/config.py

@@ -419,6 +419,12 @@
     environ=['SINGLESTOREDB_EXT_FUNC_NAME_SUFFIX'],
 )
 
+register_option(
+    'external_function.function_database', 'string', check_str, '',
+    'Database to use for the function definitions.',
+    environ=['SINGLESTOREDB_EXT_FUNC_FUNCTION_DATABASE'],
+)
+
 register_option(
     'external_function.connection', 'string', check_str,
     os.environ.get('SINGLESTOREDB_URL') or None,

singlestoredb/docstring/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Marcin Kurczewski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

singlestoredb/docstring/README.md

@@ -0,0 +1,81 @@
+docstring_parser
+================
+
+[![Build](https://github.com/rr-/docstring_parser/actions/workflows/build.yml/badge.svg)](https://github.com/rr-/docstring_parser/actions/workflows/build.yml)
+
+Parse Python docstrings. Currently support ReST, Google, Numpydoc-style and
+Epydoc docstrings.
+
+Example usage:
+
+```python
+>>> from docstring_parser import parse
+>>>
+>>>
+>>> docstring = parse(
+...     '''
+...     Short description
+...
+...     Long description spanning multiple lines
+...     - First line
+...     - Second line
+...     - Third line
+...
+...     :param name: description 1
+...     :param int priority: description 2
+...     :param str sender: description 3
+...     :raises ValueError: if name is invalid
+...     ''')
+>>>
+>>> docstring.long_description
+'Long description spanning multiple lines\n- First line\n- Second line\n- Third line'
+>>> docstring.params[1].arg_name
+'priority'
+>>> docstring.raises[0].type_name
+'ValueError'
+```
+
+Read [API Documentation](https://rr-.github.io/docstring_parser/).
+
+# Installation
+
+Installation using pip
+
+```shell
+pip install docstring_parser
+
+# or if you want to install it in a virtual environment
+
+python -m venv venv # create environment
+source venv/bin/activate # activate environment
+python -m pip install docstring_parser
+```
+
+Installation using conda
+
+
+1. Download and install miniconda or anaconda
+2. Install the package from the conda-forge channel via:
+  - `conda install -c conda-forge docstring_parser`
+  - or create a new conda environment via `conda create -n my-new-environment -c conda-forge docstring_parser`
+
+
+# Contributing
+
+To set up the project:
+```sh
+git clone https://github.com/rr-/docstring_parser.git
+cd docstring_parser
+
+python -m venv venv  # create environment
+source venv/bin/activate  # activate environment
+
+pip install -e ".[dev]"  # install as editable
+pre-commit install  # make sure pre-commit is setup
+```
+
+To run tests:
+```
+source venv/bin/activate
+pytest
+```

singlestoredb/docstring/__init__.py

@@ -0,0 +1,33 @@
+"""Parse docstrings as per Sphinx notation."""
+from .common import Docstring
+from .common import DocstringDeprecated
+from .common import DocstringMeta
+from .common import DocstringParam
+from .common import DocstringRaises
+from .common import DocstringReturns
+from .common import DocstringStyle
+from .common import ParseError
+from .common import RenderingStyle
+from .parser import compose
+from .parser import parse
+from .parser import parse_from_object
+from .util import combine_docstrings
+
+Style = DocstringStyle  # backwards compatibility
+
+__all__ = [
+    'parse',
+    'parse_from_object',
+    'combine_docstrings',
+    'compose',
+    'ParseError',
+    'Docstring',
+    'DocstringMeta',
+    'DocstringParam',
+    'DocstringRaises',
+    'DocstringReturns',
+    'DocstringDeprecated',
+    'DocstringStyle',
+    'RenderingStyle',
+    'Style',
+]

singlestoredb/docstring/attrdoc.py

@@ -0,0 +1,126 @@
+"""Attribute docstrings parsing.
+
+.. seealso:: https://peps.python.org/pep-0257/#what-is-a-docstring
+"""
+import ast
+import inspect
+import textwrap
+import typing as T
+from types import ModuleType
+
+from .common import Docstring
+from .common import DocstringParam
+
+
+def ast_get_constant_value(node: ast.AST) -> T.Any:
+    """Return the constant's value if the given node is a constant."""
+    return getattr(node, 'value')
+
+
+def ast_unparse(node: ast.AST) -> T.Optional[str]:
+    """Convert the AST node to source code as a string."""
+    if hasattr(ast, 'unparse'):
+        return ast.unparse(node)
+    # Support simple cases in Python < 3.9
+    if isinstance(node, ast.Constant):
+        return str(ast_get_constant_value(node))
+    if isinstance(node, ast.Name):
+        return node.id
+    return None
+
+
+def ast_is_literal_str(node: ast.AST) -> bool:
+    """Return True if the given node is a literal string."""
+    return (
+        isinstance(node, ast.Expr)
+        and isinstance(node.value, ast.Constant)
+        and isinstance(ast_get_constant_value(node.value), str)
+    )
+
+
+def ast_get_attribute(
+    node: ast.AST,
+) -> T.Optional[T.Tuple[str, T.Optional[str], T.Optional[str]]]:
+    """Return name, type and default if the given node is an attribute."""
+    if isinstance(node, (ast.Assign, ast.AnnAssign)):
+        target = (
+            node.targets[0] if isinstance(node, ast.Assign) else node.target
+        )
+        if isinstance(target, ast.Name):
+            type_str = None
+            if isinstance(node, ast.AnnAssign):
+                type_str = ast_unparse(node.annotation)
+            default = None
+            if node.value:
+                default = ast_unparse(node.value)
+            return target.id, type_str, default
+    return None
+
+
+class AttributeDocstrings(ast.NodeVisitor):
+    """An ast.NodeVisitor that collects attribute docstrings."""
+
+    attr_docs: T.Dict[str, T.Tuple[str, T.Optional[str], T.Optional[str]]] = {}
+    prev_attr = None
+
+    def visit(self, node: T.Any) -> None:
+        if self.prev_attr and ast_is_literal_str(node):
+            attr_name, attr_type, attr_default = self.prev_attr
+            self.attr_docs[attr_name] = (
+                ast_get_constant_value(node.value),
+                attr_type,
+                attr_default,
+            )
+        self.prev_attr = ast_get_attribute(node)
+        if isinstance(node, (ast.ClassDef, ast.Module)):
+            self.generic_visit(node)
+
+    def get_attr_docs(
+        self, component: T.Any,
+    ) -> T.Dict[str, T.Tuple[str, T.Optional[str], T.Optional[str]]]:
+        """Get attribute docstrings from the given component.
+
+        :param component: component to process (class or module)
+        :returns: for each attribute docstring, a tuple with (description,
+            type, default)
+        """
+        self.attr_docs = {}
+        self.prev_attr = None
+        try:
+            source = textwrap.dedent(inspect.getsource(component))
+        except OSError:
+            pass
+        else:
+            tree = ast.parse(source)
+            if inspect.ismodule(component):
+                self.visit(tree)
+            elif isinstance(tree, ast.Module) and isinstance(
+                tree.body[0], ast.ClassDef,
+            ):
+                self.visit(tree.body[0])
+        return self.attr_docs
+
+
+def add_attribute_docstrings(
+    obj: T.Union[type, ModuleType], docstring: Docstring,
+) -> None:
+    """Add attribute docstrings found in the object's source code.
+
+    :param obj: object from which to parse attribute docstrings
+    :param docstring: Docstring object where found attributes are added
+    :returns: list with names of added attributes
+    """
+    params = set(p.arg_name for p in docstring.params)
+    for arg_name, (description, type_name, default) in (
+        AttributeDocstrings().get_attr_docs(obj).items()
+    ):
+        if arg_name not in params:
+            param = DocstringParam(
+                args=['attribute', arg_name],
+                description=description,
+                arg_name=arg_name,
+                type_name=type_name,
+                is_optional=default is not None,
+                default=default,
+            )
+            docstring.meta.append(param)