From fcb58cecf8777e5392173c3bffb05213761359a5 Mon Sep 17 00:00:00 2001 From: codegen-bot Date: Sat, 15 Mar 2025 22:26:59 +0000 Subject: [PATCH 1/2] CG-12188: Add lazy_parse flag to Codebase constructor --- src/codegen/sdk/core/codebase_updated.py | 229 ++++++++++++ .../sdk/core/codebase_with_lazy_parse.py | 333 ++++++++++++++++++ 2 files changed, 562 insertions(+) create mode 100644 src/codegen/sdk/core/codebase_updated.py create mode 100644 src/codegen/sdk/core/codebase_with_lazy_parse.py diff --git a/src/codegen/sdk/core/codebase_updated.py b/src/codegen/sdk/core/codebase_updated.py new file mode 100644 index 000000000..fa0ea13ad --- /dev/null +++ b/src/codegen/sdk/core/codebase_updated.py @@ -0,0 +1,229 @@ +"""Codebase - main interface for Codemods to interact with the codebase""" + +import codecs +import json +import os +import re +import tempfile +from collections.abc import Generator +from contextlib import contextmanager +from functools import cached_property +from pathlib import Path +from typing import Generic, Literal, Unpack, overload + +import plotly.graph_objects as go +import rich.repr +from git import Commit as GitCommit +from git import Diff +from git.remote import PushInfoList +from github.PullRequest import PullRequest +from networkx import Graph +from openai import OpenAI +from rich.console import Console +from typing_extensions import TypeVar, deprecated + +from codegen.configs.models.codebase import CodebaseConfig, PinkMode +from codegen.configs.models.secrets import SecretsConfig +from codegen.git.repo_operator.repo_operator import RepoOperator +from codegen.git.schemas.enums import CheckoutResult, SetupOption +from codegen.git.schemas.repo_config import RepoConfig +from codegen.git.utils.pr_review import CodegenPR +from codegen.sdk._proxy import proxy_property +from codegen.sdk.ai.client import get_openai_client +from codegen.sdk.codebase.codebase_ai import generate_system_prompt, generate_tools +from codegen.sdk.codebase.codebase_context import ( + GLOBAL_FILE_IGNORE_LIST, + CodebaseContext, +) +from codegen.sdk.codebase.config import ProjectConfig, SessionOptions +from codegen.sdk.codebase.diff_lite import DiffLite +from codegen.sdk.codebase.flagging.code_flag import CodeFlag +from codegen.sdk.codebase.flagging.enums import FlagKwargs +from codegen.sdk.codebase.flagging.group import Group +from codegen.sdk.codebase.io.io import IO +from codegen.sdk.codebase.progress.progress import Progress +from codegen.sdk.codebase.span import Span +from codegen.sdk.core.assignment import Assignment +from codegen.sdk.core.class_definition import Class +from codegen.sdk.core.codeowner import CodeOwner +from codegen.sdk.core.detached_symbols.code_block import CodeBlock +from codegen.sdk.core.detached_symbols.parameter import Parameter +from codegen.sdk.core.directory import Directory +from codegen.sdk.core.export import Export +from codegen.sdk.core.external_module import ExternalModule +from codegen.sdk.core.file import File, SourceFile +from codegen.sdk.core.function import Function +from codegen.sdk.core.import_resolution import Import +from codegen.sdk.core.interface import Interface +from codegen.sdk.core.interfaces.editable import Editable +from codegen.sdk.core.interfaces.has_name import HasName +from codegen.sdk.core.symbol import Symbol +from codegen.sdk.core.type_alias import TypeAlias +from codegen.sdk.enums import NodeType, SymbolType +from codegen.sdk.extensions.sort import sort_editables +from codegen.sdk.extensions.utils import uncache_all +from codegen.sdk.output.constants import ANGULAR_STYLE +from codegen.sdk.python.assignment import PyAssignment +from codegen.sdk.python.class_definition import PyClass +from codegen.sdk.python.detached_symbols.code_block import PyCodeBlock +from codegen.sdk.python.detached_symbols.parameter import PyParameter +from codegen.sdk.python.file import PyFile +from codegen.sdk.python.function import PyFunction +from codegen.sdk.python.import_resolution import PyImport +from codegen.sdk.python.statements.import_statement import PyImportStatement +from codegen.sdk.python.symbol import PySymbol +from codegen.sdk.typescript.assignment import TSAssignment +from codegen.sdk.typescript.class_definition import TSClass +from codegen.sdk.typescript.detached_symbols.code_block import TSCodeBlock +from codegen.sdk.typescript.detached_symbols.parameter import TSParameter +from codegen.sdk.typescript.export import TSExport +from codegen.sdk.typescript.file import TSFile +from codegen.sdk.typescript.function import TSFunction +from codegen.sdk.typescript.import_resolution import TSImport +from codegen.sdk.typescript.interface import TSInterface +from codegen.sdk.typescript.statements.import_statement import TSImportStatement +from codegen.sdk.typescript.symbol import TSSymbol +from codegen.sdk.typescript.type_alias import TSTypeAlias +from codegen.shared.decorators.docs import apidoc, noapidoc, py_noapidoc +from codegen.shared.enums.programming_language import ProgrammingLanguage +from codegen.shared.exceptions.control_flow import MaxAIRequestsError +from codegen.shared.logging.get_logger import get_logger +from codegen.shared.performance.stopwatch_utils import stopwatch +from codegen.visualizations.visualization_manager import VisualizationManager + +logger = get_logger(__name__) +MAX_LINES = 10000 # Maximum number of lines of text allowed to be logged + + +TSourceFile = TypeVar("TSourceFile", bound="SourceFile", default=SourceFile) +TDirectory = TypeVar("TDirectory", bound="Directory", default=Directory) +TSymbol = TypeVar("TSymbol", bound="Symbol", default=Symbol) +TClass = TypeVar("TClass", bound="Class", default=Class) +TFunction = TypeVar("TFunction", bound="Function", default=Function) +TImport = TypeVar("TImport", bound="Import", default=Import) +TGlobalVar = TypeVar("TGlobalVar", bound="Assignment", default=Assignment) +TInterface = TypeVar("TInterface", bound="Interface", default=Interface) +TTypeAlias = TypeVar("TTypeAlias", bound="TypeAlias", default=TypeAlias) +TParameter = TypeVar("TParameter", bound="Parameter", default=Parameter) +TCodeBlock = TypeVar("TCodeBlock", bound="CodeBlock", default=CodeBlock) +TExport = TypeVar("TExport", bound="Export", default=Export) +TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment", default=Assignment) +PyGlobalVar = TypeVar("PyGlobalVar", bound="Assignment", default=Assignment) +TSDirectory = Directory[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport] +PyDirectory = Directory[PyFile, PySymbol, PyImportStatement, PyGlobalVar, PyClass, PyFunction, PyImport] + + +@apidoc +class Codebase( + Generic[ + TSourceFile, + TDirectory, + TSymbol, + TClass, + TFunction, + TImport, + TGlobalVar, + TInterface, + TTypeAlias, + TParameter, + TCodeBlock, + ] +): + """This class provides the main entrypoint for most programs to analyzing and manipulating codebases. + + Attributes: + viz: Manages visualization of the codebase graph. + repo_path: The path to the repository. + console: Manages console output for the codebase. + """ + + _op: RepoOperator + viz: VisualizationManager + repo_path: Path + console: Console + + @overload + def __init__( + self, + repo_path: None = None, + *, + language: None = None, + projects: list[ProjectConfig] | ProjectConfig, + config: CodebaseConfig | None = None, + secrets: SecretsConfig | None = None, + io: IO | None = None, + progress: Progress | None = None, + lazy_parse: bool = False, + ) -> None: ... + + @overload + def __init__( + self, + repo_path: str, + *, + language: Literal["python", "typescript"] | ProgrammingLanguage | None = None, + projects: None = None, + config: CodebaseConfig | None = None, + secrets: SecretsConfig | None = None, + io: IO | None = None, + progress: Progress | None = None, + lazy_parse: bool = False, + ) -> None: ... + + def __init__( + self, + repo_path: str | None = None, + *, + language: Literal["python", "typescript"] | ProgrammingLanguage | None = None, + projects: list[ProjectConfig] | ProjectConfig | None = None, + config: CodebaseConfig | None = None, + secrets: SecretsConfig | None = None, + io: IO | None = None, + progress: Progress | None = None, + lazy_parse: bool = False, + ) -> None: + # Sanity check inputs + if repo_path is not None and projects is not None: + msg = "Cannot specify both repo_path and projects" + raise ValueError(msg) + + if repo_path is None and projects is None: + msg = "Must specify either repo_path or projects" + raise ValueError(msg) + + if projects is not None and language is not None: + msg = "Cannot specify both projects and language. Use ProjectConfig.from_path() to create projects with a custom language." + raise ValueError(msg) + + # If projects is a single ProjectConfig, convert it to a list + if isinstance(projects, ProjectConfig): + projects = [projects] + + # Initialize project with repo_path if projects is None + if repo_path is not None: + main_project = ProjectConfig.from_path( + repo_path, + programming_language=ProgrammingLanguage(language.upper()) if language else None, + ) + projects = [main_project] + else: + main_project = projects[0] + + # Create config if not provided + if config is None: + config = CodebaseConfig() + + # Enable lazy parsing if requested + if lazy_parse: + config.exp_lazy_graph = True + + # Initialize codebase + self._op = main_project.repo_operator + self.viz = VisualizationManager(op=self._op) + self.repo_path = Path(self._op.repo_path) + self.ctx = CodebaseContext(projects, config=config, secrets=secrets, io=io, progress=progress) + self.console = Console(record=True, soft_wrap=True) + if self.ctx.config.use_pink != PinkMode.OFF: + import codegen_sdk_pink + + self._pink_codebase = codegen_sdk_pink.Codebase(self.repo_path) \ No newline at end of file diff --git a/src/codegen/sdk/core/codebase_with_lazy_parse.py b/src/codegen/sdk/core/codebase_with_lazy_parse.py new file mode 100644 index 000000000..e6b7beb68 --- /dev/null +++ b/src/codegen/sdk/core/codebase_with_lazy_parse.py @@ -0,0 +1,333 @@ +"""Codebase - main interface for Codemods to interact with the codebase""" + +import codecs +import json +import os +import re +import tempfile +from collections.abc import Generator +from contextlib import contextmanager +from functools import cached_property +from pathlib import Path +from typing import Generic, Literal, Unpack, overload + +import plotly.graph_objects as go +import rich.repr +from git import Commit as GitCommit +from git import Diff +from git.remote import PushInfoList +from github.PullRequest import PullRequest +from networkx import Graph +from openai import OpenAI +from rich.console import Console +from typing_extensions import TypeVar, deprecated + +from codegen.configs.models.codebase import CodebaseConfig, PinkMode +from codegen.configs.models.secrets import SecretsConfig +from codegen.git.repo_operator.repo_operator import RepoOperator +from codegen.git.schemas.enums import CheckoutResult, SetupOption +from codegen.git.schemas.repo_config import RepoConfig +from codegen.git.utils.pr_review import CodegenPR +from codegen.sdk._proxy import proxy_property +from codegen.sdk.ai.client import get_openai_client +from codegen.sdk.codebase.codebase_ai import generate_system_prompt, generate_tools +from codegen.sdk.codebase.codebase_context import ( + GLOBAL_FILE_IGNORE_LIST, + CodebaseContext, +) +from codegen.sdk.codebase.config import ProjectConfig, SessionOptions +from codegen.sdk.codebase.diff_lite import DiffLite +from codegen.sdk.codebase.flagging.code_flag import CodeFlag +from codegen.sdk.codebase.flagging.enums import FlagKwargs +from codegen.sdk.codebase.flagging.group import Group +from codegen.sdk.codebase.io.io import IO +from codegen.sdk.codebase.progress.progress import Progress +from codegen.sdk.codebase.span import Span +from codegen.sdk.core.assignment import Assignment +from codegen.sdk.core.class_definition import Class +from codegen.sdk.core.codeowner import CodeOwner +from codegen.sdk.core.detached_symbols.code_block import CodeBlock +from codegen.sdk.core.detached_symbols.parameter import Parameter +from codegen.sdk.core.directory import Directory +from codegen.sdk.core.export import Export +from codegen.sdk.core.external_module import ExternalModule +from codegen.sdk.core.file import File, SourceFile +from codegen.sdk.core.function import Function +from codegen.sdk.core.import_resolution import Import +from codegen.sdk.core.interface import Interface +from codegen.sdk.core.interfaces.editable import Editable +from codegen.sdk.core.interfaces.has_name import HasName +from codegen.sdk.core.symbol import Symbol +from codegen.sdk.core.type_alias import TypeAlias +from codegen.sdk.enums import NodeType, SymbolType +from codegen.sdk.extensions.sort import sort_editables +from codegen.sdk.extensions.utils import uncache_all +from codegen.sdk.output.constants import ANGULAR_STYLE +from codegen.sdk.python.assignment import PyAssignment +from codegen.sdk.python.class_definition import PyClass +from codegen.sdk.python.detached_symbols.code_block import PyCodeBlock +from codegen.sdk.python.detached_symbols.parameter import PyParameter +from codegen.sdk.python.file import PyFile +from codegen.sdk.python.function import PyFunction +from codegen.sdk.python.import_resolution import PyImport +from codegen.sdk.python.statements.import_statement import PyImportStatement +from codegen.sdk.python.symbol import PySymbol +from codegen.sdk.typescript.assignment import TSAssignment +from codegen.sdk.typescript.class_definition import TSClass +from codegen.sdk.typescript.detached_symbols.code_block import TSCodeBlock +from codegen.sdk.typescript.detached_symbols.parameter import TSParameter +from codegen.sdk.typescript.export import TSExport +from codegen.sdk.typescript.file import TSFile +from codegen.sdk.typescript.function import TSFunction +from codegen.sdk.typescript.import_resolution import TSImport +from codegen.sdk.typescript.interface import TSInterface +from codegen.sdk.typescript.statements.import_statement import TSImportStatement +from codegen.sdk.typescript.symbol import TSSymbol +from codegen.sdk.typescript.type_alias import TSTypeAlias +from codegen.shared.decorators.docs import apidoc, noapidoc, py_noapidoc +from codegen.shared.enums.programming_language import ProgrammingLanguage +from codegen.shared.exceptions.control_flow import MaxAIRequestsError +from codegen.shared.logging.get_logger import get_logger +from codegen.shared.performance.stopwatch_utils import stopwatch +from codegen.visualizations.visualization_manager import VisualizationManager + +logger = get_logger(__name__) +MAX_LINES = 10000 # Maximum number of lines of text allowed to be logged + + +TSourceFile = TypeVar("TSourceFile", bound="SourceFile", default=SourceFile) +TDirectory = TypeVar("TDirectory", bound="Directory", default=Directory) +TSymbol = TypeVar("TSymbol", bound="Symbol", default=Symbol) +TClass = TypeVar("TClass", bound="Class", default=Class) +TFunction = TypeVar("TFunction", bound="Function", default=Function) +TImport = TypeVar("TImport", bound="Import", default=Import) +TGlobalVar = TypeVar("TGlobalVar", bound="Assignment", default=Assignment) +TInterface = TypeVar("TInterface", bound="Interface", default=Interface) +TTypeAlias = TypeVar("TTypeAlias", bound="TypeAlias", default=TypeAlias) +TParameter = TypeVar("TParameter", bound="Parameter", default=Parameter) +TCodeBlock = TypeVar("TCodeBlock", bound="CodeBlock", default=CodeBlock) +TExport = TypeVar("TExport", bound="Export", default=Export) +TSGlobalVar = TypeVar("TSGlobalVar", bound="Assignment", default=Assignment) +PyGlobalVar = TypeVar("PyGlobalVar", bound="Assignment", default=Assignment) +TSDirectory = Directory[TSFile, TSSymbol, TSImportStatement, TSGlobalVar, TSClass, TSFunction, TSImport] +PyDirectory = Directory[PyFile, PySymbol, PyImportStatement, PyGlobalVar, PyClass, PyFunction, PyImport] + + +@apidoc +class Codebase( + Generic[ + TSourceFile, + TDirectory, + TSymbol, + TClass, + TFunction, + TImport, + TGlobalVar, + TInterface, + TTypeAlias, + TParameter, + TCodeBlock, + ] +): + """This class provides the main entrypoint for most programs to analyzing and manipulating codebases. + + Attributes: + viz: Manages visualization of the codebase graph. + repo_path: The path to the repository. + console: Manages console output for the codebase. + """ + + _op: RepoOperator + viz: VisualizationManager + repo_path: Path + console: Console + + @overload + def __init__( + self, + repo_path: None = None, + *, + language: None = None, + projects: list[ProjectConfig] | ProjectConfig, + config: CodebaseConfig | None = None, + secrets: SecretsConfig | None = None, + io: IO | None = None, + progress: Progress | None = None, + lazy_parse: bool = False, + ) -> None: ... + + @overload + def __init__( + self, + repo_path: str, + *, + language: Literal["python", "typescript"] | ProgrammingLanguage | None = None, + projects: None = None, + config: CodebaseConfig | None = None, + secrets: SecretsConfig | None = None, + io: IO | None = None, + progress: Progress | None = None, + lazy_parse: bool = False, + ) -> None: ... + + def __init__( + self, + repo_path: str | None = None, + *, + language: Literal["python", "typescript"] | ProgrammingLanguage | None = None, + projects: list[ProjectConfig] | ProjectConfig | None = None, + config: CodebaseConfig | None = None, + secrets: SecretsConfig | None = None, + io: IO | None = None, + progress: Progress | None = None, + lazy_parse: bool = False, + ) -> None: + # Sanity check inputs + if repo_path is not None and projects is not None: + msg = "Cannot specify both repo_path and projects" + raise ValueError(msg) + + if repo_path is None and projects is None: + msg = "Must specify either repo_path or projects" + raise ValueError(msg) + + if projects is not None and language is not None: + msg = "Cannot specify both projects and language. Use ProjectConfig.from_path() to create projects with a custom language." + raise ValueError(msg) + + # If projects is a single ProjectConfig, convert it to a list + if isinstance(projects, ProjectConfig): + projects = [projects] + + # Initialize project with repo_path if projects is None + if repo_path is not None: + main_project = ProjectConfig.from_path( + repo_path, + programming_language=ProgrammingLanguage(language.upper()) if language else None, + ) + projects = [main_project] + else: + main_project = projects[0] + + # Create config if not provided + if config is None: + config = CodebaseConfig() + + # Enable lazy parsing if requested + if lazy_parse: + config.exp_lazy_graph = True + + # Initialize codebase + self._op = main_project.repo_operator + self.viz = VisualizationManager(op=self._op) + self.repo_path = Path(self._op.repo_path) + self.ctx = CodebaseContext(projects, config=config, secrets=secrets, io=io, progress=progress) + self.console = Console(record=True, soft_wrap=True) + if self.ctx.config.use_pink != PinkMode.OFF: + import codegen_sdk_pink + + self._pink_codebase = codegen_sdk_pink.Codebase(self.repo_path) + + @noapidoc + def __str__(self) -> str: + return f"" + + @noapidoc + def __repr__(self): + return str(self) + + def __rich_repr__(self) -> rich.repr.Result: + yield "repo", self.ctx.repo_name + yield "nodes", len(self.ctx.nodes) + yield "edges", len(self.ctx.edges) + + __rich_repr__.angular = ANGULAR_STYLE + + @property + @deprecated("Please do not use the local repo operator directly") + @noapidoc + def op(self) -> RepoOperator: + return self._op + + @property + def github(self) -> RepoOperator: + """Access GitHub operations through the repo operator. + + This property provides access to GitHub operations like creating PRs, + working with branches, commenting on PRs, etc. The implementation is built + on top of PyGitHub (python-github library) and provides a simplified interface + for common GitHub operations. + + Returns: + RepoOperator: The repo operator instance that handles GitHub operations. + """ + return self._op + + #################################################################################################################### + # SIMPLE META + #################################################################################################################### + + @property + def name(self) -> str: + """The name of the repository.""" + return self.ctx.repo_name + + @property + def language(self) -> ProgrammingLanguage: + """The programming language of the repository.""" + return self.ctx.programming_language + + #################################################################################################################### + # NODES + #################################################################################################################### + + @noapidoc + def _symbols(self, symbol_type: SymbolType | None = None) -> list[TSymbol | TClass | TFunction | TGlobalVar]: + matches: list[Symbol] = self.ctx.get_nodes(NodeType.SYMBOL) + return [x for x in matches if x.is_top_level and (symbol_type is None or x.symbol_type == symbol_type)] + + # =====[ Node Types ]===== + @overload + def files(self, *, extensions: list[str]) -> list[File]: ... + @overload + def files(self, *, extensions: Literal["*"]) -> list[File]: ... + @overload + def files(self, *, extensions: None = ...) -> list[TSourceFile]: ... + @proxy_property + def files(self, *, extensions: list[str] | Literal["*"] | None = None) -> list[TSourceFile] | list[File]: + """A list property that returns all files in the codebase. + + By default, this only returns source files. Setting `extensions='*'` will return all files in the codebase, and + `extensions=[...]` will return all files with the specified extensions. + + For Python and Typescript repos WITH file parsing enabled, + `extensions='*'` is REQUIRED for listing all non source code files. + Or else, codebase.files will ONLY return source files (e.g. .py, .ts). + + For repos with file parsing disabled or repos with other languages, this will return all files in the codebase. + + Returns all Files in the codebase, sorted alphabetically. For Python codebases, returns PyFiles (python files). + For Typescript codebases, returns TSFiles (typescript files). + + Returns: + list[TSourceFile]: A sorted list of source files in the codebase. + """ + if self.ctx.config.use_pink == PinkMode.ALL_FILES: + return self._pink_codebase.files + if extensions is None and len(self.ctx.get_nodes(NodeType.FILE)) > 0: + # If extensions is None AND there is at least one file in the codebase (This checks for unsupported languages or parse-off repos), + # Return all source files + files = self.ctx.get_nodes(NodeType.FILE) + elif isinstance(extensions, str) and extensions != "*": + msg = "extensions must be a list of extensions or '*'" + raise ValueError(msg) + else: + files = [] + # Get all files with the specified extensions + for filepath, _ in self._op.iter_files( + extensions=None if extensions == "*" else extensions, + ignore_list=GLOBAL_FILE_IGNORE_LIST, + ): + files.append(self.get_file(filepath, optional=False)) + # Sort files alphabetically + return sort_editables(files, alphabetical=True, dedupe=False) \ No newline at end of file From bea4ab3cb4a484b611b1daae307344271cf70afb Mon Sep 17 00:00:00 2001 From: "codegen-sh[bot]" <131295404+codegen-sh[bot]@users.noreply.github.com> Date: Sat, 15 Mar 2025 22:27:43 +0000 Subject: [PATCH 2/2] Automated pre-commit update --- src/codegen/sdk/core/codebase_updated.py | 63 +++---------------- .../sdk/core/codebase_with_lazy_parse.py | 51 ++------------- 2 files changed, 13 insertions(+), 101 deletions(-) diff --git a/src/codegen/sdk/core/codebase_updated.py b/src/codegen/sdk/core/codebase_updated.py index fa0ea13ad..a4b9b54b2 100644 --- a/src/codegen/sdk/core/codebase_updated.py +++ b/src/codegen/sdk/core/codebase_updated.py @@ -1,94 +1,47 @@ """Codebase - main interface for Codemods to interact with the codebase""" -import codecs -import json -import os -import re -import tempfile -from collections.abc import Generator -from contextlib import contextmanager -from functools import cached_property from pathlib import Path -from typing import Generic, Literal, Unpack, overload - -import plotly.graph_objects as go -import rich.repr -from git import Commit as GitCommit -from git import Diff -from git.remote import PushInfoList -from github.PullRequest import PullRequest -from networkx import Graph -from openai import OpenAI +from typing import Generic, Literal, overload + from rich.console import Console -from typing_extensions import TypeVar, deprecated +from typing_extensions import TypeVar from codegen.configs.models.codebase import CodebaseConfig, PinkMode from codegen.configs.models.secrets import SecretsConfig from codegen.git.repo_operator.repo_operator import RepoOperator -from codegen.git.schemas.enums import CheckoutResult, SetupOption -from codegen.git.schemas.repo_config import RepoConfig -from codegen.git.utils.pr_review import CodegenPR -from codegen.sdk._proxy import proxy_property -from codegen.sdk.ai.client import get_openai_client -from codegen.sdk.codebase.codebase_ai import generate_system_prompt, generate_tools from codegen.sdk.codebase.codebase_context import ( - GLOBAL_FILE_IGNORE_LIST, CodebaseContext, ) -from codegen.sdk.codebase.config import ProjectConfig, SessionOptions -from codegen.sdk.codebase.diff_lite import DiffLite -from codegen.sdk.codebase.flagging.code_flag import CodeFlag -from codegen.sdk.codebase.flagging.enums import FlagKwargs -from codegen.sdk.codebase.flagging.group import Group +from codegen.sdk.codebase.config import ProjectConfig from codegen.sdk.codebase.io.io import IO from codegen.sdk.codebase.progress.progress import Progress -from codegen.sdk.codebase.span import Span from codegen.sdk.core.assignment import Assignment from codegen.sdk.core.class_definition import Class -from codegen.sdk.core.codeowner import CodeOwner from codegen.sdk.core.detached_symbols.code_block import CodeBlock from codegen.sdk.core.detached_symbols.parameter import Parameter from codegen.sdk.core.directory import Directory from codegen.sdk.core.export import Export -from codegen.sdk.core.external_module import ExternalModule -from codegen.sdk.core.file import File, SourceFile +from codegen.sdk.core.file import SourceFile from codegen.sdk.core.function import Function from codegen.sdk.core.import_resolution import Import from codegen.sdk.core.interface import Interface -from codegen.sdk.core.interfaces.editable import Editable -from codegen.sdk.core.interfaces.has_name import HasName from codegen.sdk.core.symbol import Symbol from codegen.sdk.core.type_alias import TypeAlias -from codegen.sdk.enums import NodeType, SymbolType -from codegen.sdk.extensions.sort import sort_editables -from codegen.sdk.extensions.utils import uncache_all -from codegen.sdk.output.constants import ANGULAR_STYLE -from codegen.sdk.python.assignment import PyAssignment from codegen.sdk.python.class_definition import PyClass -from codegen.sdk.python.detached_symbols.code_block import PyCodeBlock -from codegen.sdk.python.detached_symbols.parameter import PyParameter from codegen.sdk.python.file import PyFile from codegen.sdk.python.function import PyFunction from codegen.sdk.python.import_resolution import PyImport from codegen.sdk.python.statements.import_statement import PyImportStatement from codegen.sdk.python.symbol import PySymbol -from codegen.sdk.typescript.assignment import TSAssignment from codegen.sdk.typescript.class_definition import TSClass -from codegen.sdk.typescript.detached_symbols.code_block import TSCodeBlock -from codegen.sdk.typescript.detached_symbols.parameter import TSParameter -from codegen.sdk.typescript.export import TSExport from codegen.sdk.typescript.file import TSFile from codegen.sdk.typescript.function import TSFunction from codegen.sdk.typescript.import_resolution import TSImport -from codegen.sdk.typescript.interface import TSInterface from codegen.sdk.typescript.statements.import_statement import TSImportStatement from codegen.sdk.typescript.symbol import TSSymbol -from codegen.sdk.typescript.type_alias import TSTypeAlias -from codegen.shared.decorators.docs import apidoc, noapidoc, py_noapidoc +from codegen.shared.decorators.docs import apidoc from codegen.shared.enums.programming_language import ProgrammingLanguage -from codegen.shared.exceptions.control_flow import MaxAIRequestsError from codegen.shared.logging.get_logger import get_logger -from codegen.shared.performance.stopwatch_utils import stopwatch from codegen.visualizations.visualization_manager import VisualizationManager logger = get_logger(__name__) @@ -212,7 +165,7 @@ def __init__( # Create config if not provided if config is None: config = CodebaseConfig() - + # Enable lazy parsing if requested if lazy_parse: config.exp_lazy_graph = True @@ -226,4 +179,4 @@ def __init__( if self.ctx.config.use_pink != PinkMode.OFF: import codegen_sdk_pink - self._pink_codebase = codegen_sdk_pink.Codebase(self.repo_path) \ No newline at end of file + self._pink_codebase = codegen_sdk_pink.Codebase(self.repo_path) diff --git a/src/codegen/sdk/core/codebase_with_lazy_parse.py b/src/codegen/sdk/core/codebase_with_lazy_parse.py index e6b7beb68..c4a0225cc 100644 --- a/src/codegen/sdk/core/codebase_with_lazy_parse.py +++ b/src/codegen/sdk/core/codebase_with_lazy_parse.py @@ -1,94 +1,53 @@ """Codebase - main interface for Codemods to interact with the codebase""" -import codecs -import json -import os -import re -import tempfile -from collections.abc import Generator -from contextlib import contextmanager -from functools import cached_property from pathlib import Path -from typing import Generic, Literal, Unpack, overload +from typing import Generic, Literal, overload -import plotly.graph_objects as go import rich.repr -from git import Commit as GitCommit -from git import Diff -from git.remote import PushInfoList -from github.PullRequest import PullRequest -from networkx import Graph -from openai import OpenAI from rich.console import Console from typing_extensions import TypeVar, deprecated from codegen.configs.models.codebase import CodebaseConfig, PinkMode from codegen.configs.models.secrets import SecretsConfig from codegen.git.repo_operator.repo_operator import RepoOperator -from codegen.git.schemas.enums import CheckoutResult, SetupOption -from codegen.git.schemas.repo_config import RepoConfig -from codegen.git.utils.pr_review import CodegenPR from codegen.sdk._proxy import proxy_property -from codegen.sdk.ai.client import get_openai_client -from codegen.sdk.codebase.codebase_ai import generate_system_prompt, generate_tools from codegen.sdk.codebase.codebase_context import ( GLOBAL_FILE_IGNORE_LIST, CodebaseContext, ) -from codegen.sdk.codebase.config import ProjectConfig, SessionOptions -from codegen.sdk.codebase.diff_lite import DiffLite -from codegen.sdk.codebase.flagging.code_flag import CodeFlag -from codegen.sdk.codebase.flagging.enums import FlagKwargs -from codegen.sdk.codebase.flagging.group import Group +from codegen.sdk.codebase.config import ProjectConfig from codegen.sdk.codebase.io.io import IO from codegen.sdk.codebase.progress.progress import Progress -from codegen.sdk.codebase.span import Span from codegen.sdk.core.assignment import Assignment from codegen.sdk.core.class_definition import Class -from codegen.sdk.core.codeowner import CodeOwner from codegen.sdk.core.detached_symbols.code_block import CodeBlock from codegen.sdk.core.detached_symbols.parameter import Parameter from codegen.sdk.core.directory import Directory from codegen.sdk.core.export import Export -from codegen.sdk.core.external_module import ExternalModule from codegen.sdk.core.file import File, SourceFile from codegen.sdk.core.function import Function from codegen.sdk.core.import_resolution import Import from codegen.sdk.core.interface import Interface -from codegen.sdk.core.interfaces.editable import Editable -from codegen.sdk.core.interfaces.has_name import HasName from codegen.sdk.core.symbol import Symbol from codegen.sdk.core.type_alias import TypeAlias from codegen.sdk.enums import NodeType, SymbolType from codegen.sdk.extensions.sort import sort_editables -from codegen.sdk.extensions.utils import uncache_all from codegen.sdk.output.constants import ANGULAR_STYLE -from codegen.sdk.python.assignment import PyAssignment from codegen.sdk.python.class_definition import PyClass -from codegen.sdk.python.detached_symbols.code_block import PyCodeBlock -from codegen.sdk.python.detached_symbols.parameter import PyParameter from codegen.sdk.python.file import PyFile from codegen.sdk.python.function import PyFunction from codegen.sdk.python.import_resolution import PyImport from codegen.sdk.python.statements.import_statement import PyImportStatement from codegen.sdk.python.symbol import PySymbol -from codegen.sdk.typescript.assignment import TSAssignment from codegen.sdk.typescript.class_definition import TSClass -from codegen.sdk.typescript.detached_symbols.code_block import TSCodeBlock -from codegen.sdk.typescript.detached_symbols.parameter import TSParameter -from codegen.sdk.typescript.export import TSExport from codegen.sdk.typescript.file import TSFile from codegen.sdk.typescript.function import TSFunction from codegen.sdk.typescript.import_resolution import TSImport -from codegen.sdk.typescript.interface import TSInterface from codegen.sdk.typescript.statements.import_statement import TSImportStatement from codegen.sdk.typescript.symbol import TSSymbol -from codegen.sdk.typescript.type_alias import TSTypeAlias -from codegen.shared.decorators.docs import apidoc, noapidoc, py_noapidoc +from codegen.shared.decorators.docs import apidoc, noapidoc from codegen.shared.enums.programming_language import ProgrammingLanguage -from codegen.shared.exceptions.control_flow import MaxAIRequestsError from codegen.shared.logging.get_logger import get_logger -from codegen.shared.performance.stopwatch_utils import stopwatch from codegen.visualizations.visualization_manager import VisualizationManager logger = get_logger(__name__) @@ -212,7 +171,7 @@ def __init__( # Create config if not provided if config is None: config = CodebaseConfig() - + # Enable lazy parsing if requested if lazy_parse: config.exp_lazy_graph = True @@ -330,4 +289,4 @@ def files(self, *, extensions: list[str] | Literal["*"] | None = None) -> list[T ): files.append(self.get_file(filepath, optional=False)) # Sort files alphabetically - return sort_editables(files, alphabetical=True, dedupe=False) \ No newline at end of file + return sort_editables(files, alphabetical=True, dedupe=False)