Add GitRepo as SAST requirement and provide a method to download the repo

Author: nolliv22

codesectools/sasts/SemgrepCE/sast.py

@@ -6,11 +6,10 @@
 
 from pathlib import Path
 
-from codesectools.datasets.SemgrepCERules.dataset import SemgrepCERules
 from codesectools.sasts.core.sast.properties import SASTProperties
 from codesectools.sasts.core.sast.requirements import (
     Binary,
-    DatasetCache,
+    GitRepo,
     SASTRequirements,
 )
 from codesectools.sasts.core.sast.sast import SAST
@@ -43,13 +42,18 @@ class SemgrepCESAST(SAST):
         full_reqs=[
             Binary("semgrep", url="https://semgrep.dev/docs/getting-started/quickstart")
         ],
-        partial_reqs=[DatasetCache("SemgrepCERules")],
+        partial_reqs=[
+            GitRepo(
+                name="semgrep-rules",
+                repo_url="https://github.com/semgrep/semgrep-rules.git",
+            )
+        ],
     )
     commands = [
         [
             "semgrep",
             "scan",
-            f"--config={str(USER_CACHE_DIR / SemgrepCERules.name / '{lang}')}",
+            f"--config={str(USER_CACHE_DIR / 'semgrep-rules' / '{lang}')}",
             "--metrics=off",
             "--json-output=semgrep_output.json",
         ]

codesectools/sasts/core/cli.py

@@ -27,7 +27,7 @@
 
 
 class CLIFactory:
-    """A factory to generate a standard set of CLI commands for a SAST tool.
+    """Provide a factory to generate a standard set of CLI commands for a SAST tool.
 
     Attributes:
         cli (typer.Typer): The `typer` application to which commands will be added.
@@ -109,7 +109,7 @@ def install() -> None:
             install_help = ""
             sast_reqs = self.sast.requirements
             missing_reqs = sast_reqs.get_missing()
-            for req in sast_reqs.full_reqs + sast_reqs.partial_reqs:
+            for req in sast_reqs.full + sast_reqs.partial:
                 install_help += (
                     f"{'❌' if req in missing_reqs else '✅'} [b]{req}[/b]\n"
                 )

codesectools/sasts/core/sast/requirements.py

@@ -4,10 +4,13 @@
 from abc import ABC, abstractmethod
 from typing import Literal
 
-from codesectools.datasets import DATASETS_ALL
-from codesectools.utils import (
-    USER_CONFIG_DIR,
-)
+import typer
+from git import Repo
+from rich import print
+from rich.panel import Panel
+from rich.progress import Progress
+
+from codesectools.utils import USER_CACHE_DIR, USER_CONFIG_DIR
 
 
 class SASTRequirement(ABC):
@@ -24,7 +27,7 @@ def __init__(
 
         Args:
             name: The name of the requirement.
-            instruction: A short instruction on how to fulfill the requirement.
+            instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
             doc: A flag indicating if the instruction is available in the documentaton.
 
@@ -44,6 +47,36 @@ def __repr__(self) -> str:
         return f"{self.__class__.__name__}({self.name})"
 
 
+class DownloadableRequirement(SASTRequirement):
+    """Represent a SAST requirement that can be downloaded automatically."""
+
+    def __init__(
+        self,
+        name: str,
+        instruction: str | None = None,
+        url: str | None = None,
+        doc: bool = False,
+    ) -> None:
+        """Initialize a DownloadableRequirement instance.
+
+        Sets a standard instruction message on how to download the requirement using the CLI.
+
+        Args:
+            name: The name of the requirement.
+            instruction: A short instruction on how to download the requirement.
+            url: A URL for more detailed instructions.
+            doc: A flag indicating if the instruction is available in the documentaton.
+
+        """
+        instruction = f"cstools download {name}"
+        super().__init__(name, instruction, url, doc)
+
+    @abstractmethod
+    def download(self, **kwargs: dict) -> None:
+        """Download the requirement."""
+        pass
+
+
 class Config(SASTRequirement):
     """Represent a configuration file requirement for a SAST tool."""
 
@@ -58,7 +91,7 @@ def __init__(
 
         Args:
             name: The name of the requirement.
-            instruction: A short instruction on how to fulfill the requirement.
+            instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
             doc: A flag indicating if this is a documentation-only requirement.
 
@@ -84,7 +117,7 @@ def __init__(
 
         Args:
             name: The name of the requirement.
-            instruction: A short instruction on how to fulfill the requirement.
+            instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
             doc: A flag indicating if this is a documentation-only requirement.
 
@@ -96,31 +129,61 @@ def is_fulfilled(self, **kwargs: dict) -> bool:
         return bool(shutil.which(self.name))
 
 
-class DatasetCache(SASTRequirement):
-    """Represent a dataset cache requirement for a SAST tool."""
+class GitRepo(DownloadableRequirement):
+    """Represent a Git repository requirement that can be downloaded."""
 
     def __init__(
         self,
         name: str,
+        repo_url: str,
         instruction: str | None = None,
         url: str | None = None,
         doc: bool = False,
     ) -> None:
-        """Initialize a DatasetCache instance.
+        """Initialize a GitRepo requirement instance.
 
         Args:
             name: The name of the requirement.
-            instruction: A short instruction on how to fulfill the requirement.
+            repo_url: The URL of the Git repository to clone.
+            instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
-            doc: A flag indicating if this is a documentation-only requirement.
+            doc: A flag indicating if the instruction is available in the documentaton.
 
         """
-        instruction = f"cstools dataset download {name}"
         super().__init__(name, instruction, url, doc)
+        self.repo_url = repo_url
+        self.directory = USER_CACHE_DIR / self.name
 
     def is_fulfilled(self, **kwargs: dict) -> bool:
-        """Check if the dataset is cached locally."""
-        return DATASETS_ALL[self.name].is_cached()
+        """Check if the Git repository has been cloned."""
+        return (self.directory / ".complete").is_file()
+
+    def download(self, **kwargs: dict) -> None:
+        """Prompt for license agreement and clone the Git repository."""
+        panel = Panel(
+            f"""Git repository:\t[b]{self.name}[/b]
+Repository URL:\t[u]{self.repo_url}[/u]
+
+Please review the license of the repository at the URL above.
+By proceeding, you agree to abide by its terms.""",
+            title="[b]License Agreement[/b]",
+        )
+        print(panel)
+
+        agreed = typer.confirm("Do you accept the license terms and wish to proceed?")
+        if not agreed:
+            print("[red]License agreement declined. Download aborted.[/red]")
+            raise typer.Exit(code=1)
+
+        with Progress() as progress:
+            progress.add_task(f"Cloning repository [b]{self.name}[/b]...", total=None)
+            Repo.clone_from(
+                self.repo_url,
+                self.directory,
+                depth=1,
+            )
+        (self.directory / ".complete").write_bytes(b"\x42")
+        print(f"[b]{self.name}[/b] has been downloaded at {self.directory}.")
 
 
 class SASTRequirements:
@@ -137,25 +200,26 @@ def __init__(
 
         """
         self.name = None
-        self.full_reqs = full_reqs
-        self.partial_reqs = partial_reqs
+        self.full = full_reqs
+        self.partial = partial_reqs
+        self.all = full_reqs + partial_reqs
 
     def get_status(self) -> Literal["full"] | Literal["partial"] | Literal["none"]:
         """Determine the operational status (full, partial, none) based on fulfilled requirements."""
         # full: can run sast analysis and result parsing
         # partial: can run result parsing
         # none: nothing
         status = "none"
-        if all(req.is_fulfilled(sast_name=self.name) for req in self.partial_reqs):
+        if all(req.is_fulfilled(sast_name=self.name) for req in self.partial):
             status = "partial"
-            if all(req.is_fulfilled(sast_name=self.name) for req in self.full_reqs):
+            if all(req.is_fulfilled(sast_name=self.name) for req in self.full):
                 status = "full"
         return status
 
     def get_missing(self) -> list[SASTRequirement]:
         """Get a list of all unfulfilled requirements."""
         missing = []
-        for req in self.full_reqs + self.partial_reqs:
+        for req in self.all:
             if not req.is_fulfilled(sast_name=self.name):
                 missing.append(req)
         return missing

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "CodeSecTools"
-version = "0.4.1"
+version = "0.5.0"
 description = "A framework for code security that provides abstractions for static analysis tools and datasets to support their integration, testing, and evaluation."
 readme = "README.md"
 license = "AGPL-3.0-only"