feat(api): Add the possibility to save results from external source (#112)

* feat(api): Add the possibility to save results from external source

* chore: fix linting issues

* feat(batch): add batch import result feature

* chore: fix linting and poetry issues

* chore(api): Bump API version to 3.8.0a0

* fix(api): fix worker to handle multiple queues

* fix(scraper): To work with newest version of chromium

Author: vvatelot

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 {
     "name": "Ecoindex python full stack dev container",
     "image": "mcr.microsoft.com/devcontainers/python:3.12",
-    "postCreateCommand": "pipx install poetry && poetry self add poetry-multiproject-plugin && poetry self add poetry-polylith-plugin",
+    "postCreateCommand": "pipx install poetry==1.8.5 && poetry self add poetry-multiproject-plugin && poetry self add poetry-polylith-plugin",
     "features": {
         "ghcr.io/audacioustux/devcontainers/taskfile": {},
         "ghcr.io/devcontainers/features/docker-in-docker:2": {

.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+
+        {
+            "name": "Python Debugger: FastAPI",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "uvicorn",
+            "args": [
+                "ecoindex.backend.main:app",
+                "--reload"
+            ],
+            "jinja": true
+        }
+    ]
+}

bases/ecoindex/backend/VERSION

@@ -1 +1 @@
-3.7.0
+3.8.0a0

bases/ecoindex/backend/dependencies/validation.py

@@ -0,0 +1,26 @@
+from typing import Annotated
+
+from ecoindex.config.settings import Settings
+from fastapi import Header, HTTPException, status
+
+
+def validate_api_key_batch(
+    api_key: Annotated[
+        str,
+        Header(alias="X-Api-Key"),
+    ],
+):
+    if not api_key:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid API key",
+        )
+
+    for authorized_api_key in Settings().API_KEYS_BATCH:
+        if api_key == authorized_api_key["key"]:
+            return authorized_api_key
+
+    raise HTTPException(
+        status_code=status.HTTP_403_FORBIDDEN,
+        detail="Invalid API key",
+    )

bases/ecoindex/backend/routers/bff.py

@@ -104,5 +104,5 @@ async def get_latest_result_redirect(
         )
 
     return RedirectResponse(
-        url=f"{Settings().FRONTEND_BASE_URL}/resultat/?id={latest_result.latest_result.id}"
+        url=f"{Settings().FRONTEND_BASE_URL}/resultat/?id={latest_result.latest_result.id}"  # type: ignore
     )

bases/ecoindex/backend/routers/ecoindex.py

@@ -71,8 +71,9 @@ async def get_ecoindex_analysis_list(
         page=pagination.page,
         size=pagination.size,
         sort_params=await get_sort_parameters(
-            query_params=sort, model=ApiEcoindex
-        ),  # type: ignore
+            query_params=sort,
+            model=ApiEcoindex,  # type: ignore
+        ),
     )
     total_results = await get_count_analysis_db(
         session=session,

bases/ecoindex/backend/routers/tasks.py

@@ -1,19 +1,22 @@
 from json import loads
+from typing import Annotated
 
 import requests
 from celery.result import AsyncResult
+from ecoindex.backend.dependencies.validation import validate_api_key_batch
 from ecoindex.backend.models.dependencies_parameters.id import IdParameter
 from ecoindex.backend.utils import check_quota
 from ecoindex.config.settings import Settings
 from ecoindex.database.engine import get_session
+from ecoindex.database.models import ApiEcoindexes
 from ecoindex.models import WebPage
 from ecoindex.models.enums import TaskStatus
 from ecoindex.models.response_examples import (
     example_daily_limit_response,
     example_host_unreachable,
 )
-from ecoindex.models.tasks import QueueTaskApi, QueueTaskResult
-from ecoindex.worker.tasks import ecoindex_task
+from ecoindex.models.tasks import QueueTaskApi, QueueTaskApiBatch, QueueTaskResult
+from ecoindex.worker.tasks import ecoindex_batch_import_task, ecoindex_task
 from ecoindex.worker_component import app as task_app
 from fastapi import APIRouter, Depends, HTTPException, Response, status
 from fastapi.params import Body
@@ -37,18 +40,23 @@
 )
 async def add_ecoindex_analysis_task(
     response: Response,
-    web_page: WebPage = Body(
-        default=...,
-        title="Web page to analyze defined by its url and its screen resolution",
-        example=WebPage(url="https://www.ecoindex.fr", width=1920, height=1080),
-    ),
+    web_page: Annotated[
+        WebPage,
+        Body(
+            default=...,
+            title="Web page to analyze defined by its url and its screen resolution",
+            example=WebPage(url="https://www.ecoindex.fr", width=1920, height=1080),
+        ),
+    ],
     session: AsyncSession = Depends(get_session),
 ) -> str:
     if Settings().DAILY_LIMIT_PER_HOST:
         remaining_quota = await check_quota(
             session=session, host=web_page.get_url_host()
         )
-        response.headers["X-Remaining-Daily-Requests"] = str(remaining_quota - 1)
+
+        if remaining_quota:
+            response.headers["X-Remaining-Daily-Requests"] = str(remaining_quota - 1)
 
     if (
         Settings().EXCLUDED_HOSTS
@@ -63,13 +71,12 @@ async def add_ecoindex_analysis_task(
         r = requests.head(url=web_page.url, timeout=5)
         r.raise_for_status()
     except Exception:
-        print(f"The URL {web_page.url} is not reachable")
         raise HTTPException(
             status_code=521,
             detail=f"The URL {web_page.url} is unreachable. Are you really sure of this url? 🤔",
         )
 
-    task_result = ecoindex_task.delay(
+    task_result = ecoindex_task.delay(  # type: ignore
         url=str(web_page.url), width=web_page.width, height=web_page.height
     )
 
@@ -93,7 +100,7 @@ async def get_ecoindex_analysis_task_by_id(
     t = AsyncResult(id=str(id), app=task_app)
 
     task_response = QueueTaskApi(
-        id=t.id,
+        id=str(t.id),
         status=t.state,
     )
 
@@ -125,3 +132,63 @@ async def delete_ecoindex_analysis_task_by_id(
     res = task_app.control.revoke(id, terminate=True, signal="SIGKILL")
 
     return res
+
+
+@router.post(
+    name="Save ecoindex analysis from external source in batch mode",
+    path="/batch",
+    response_description="Identifier of the task that has been created in queue",
+    responses={
+        status.HTTP_201_CREATED: {"model": str},
+        status.HTTP_403_FORBIDDEN: {"model": str},
+    },
+    description="This save ecoindex analysis from external source in batch mode. Limited to 100 entries at a time",
+    status_code=status.HTTP_201_CREATED,
+)
+async def add_ecoindex_analysis_task_batch(
+    results: Annotated[
+        ApiEcoindexes,
+        Body(
+            default=...,
+            title="List of ecoindex analysis results to save",
+            example=[],
+            min_length=1,
+            max_length=100,
+        ),
+    ],
+    batch_key: str = Depends(validate_api_key_batch),
+):
+    task_result = ecoindex_batch_import_task.delay(  # type: ignore
+        results=[result.model_dump() for result in results],
+        source=batch_key["source"],  # type: ignore
+    )
+
+    return task_result.id
+
+
+@router.get(
+    name="Get ecoindex analysis batch task by id",
+    path="/batch/{id}",
+    responses={
+        status.HTTP_200_OK: {"model": QueueTaskApiBatch},
+        status.HTTP_425_TOO_EARLY: {"model": QueueTaskApiBatch},
+    },
+    response_description="Get one ecoindex batch task result by its id",
+    description="This returns an ecoindex batch task result given by its unique identifier",
+)
+async def get_ecoindex_analysis_batch_task_by_id(
+    response: Response,
+    id: IdParameter,
+    _: str = Depends(validate_api_key_batch),
+) -> QueueTaskApiBatch:
+    t = AsyncResult(id=str(id), app=task_app)
+
+    task_response = QueueTaskApiBatch(
+        id=str(t.id),
+        status=t.state,
+    )
+
+    if t.state == TaskStatus.PENDING:
+        response.status_code = status.HTTP_425_TOO_EARLY
+
+    return task_response

bases/ecoindex/backend/utils/__init__.py

@@ -19,7 +19,7 @@ async def format_exception_response(exception: Exception) -> ExceptionResponse:
     return ExceptionResponse(
         exception=type(exception).__name__,
         args=[arg for arg in exception.args if arg] if exception.args else [],
-        message=exception.msg if hasattr(exception, "msg") else None,
+        message=exception.msg if hasattr(exception, "msg") else None,  # type: ignore
     )
 
 
@@ -45,7 +45,7 @@ async def get_sort_parameters(query_params: list[str], model: BaseModel) -> list
     result = []
 
     for query_param in query_params:
-        pattern = re.compile("^\w+:(asc|desc)$")
+        pattern = re.compile("^\w+:(asc|desc)$")  # type: ignore
 
         if not re.fullmatch(pattern, query_param):
             validation_error.append(
@@ -67,8 +67,9 @@ async def get_sort_parameters(query_params: list[str], model: BaseModel) -> list
                     "type": "value_error.sort",
                 }
             )
+            continue
 
-        result.append(Sort(clause=sort_params[0], sort=sort_params[1]))
+        result.append(Sort(clause=sort_params[0], sort=sort_params[1]))  # type: ignore
 
     if validation_error:
         raise HTTPException(
@@ -94,7 +95,7 @@ async def check_quota(
         raise QuotaExceededException(
             limit=Settings().DAILY_LIMIT_PER_HOST,
             host=host,
-            latest_result=loads(latest_result.model_dump_json()),
+            latest_result=loads(latest_result.model_dump_json() or "{}"),  # type: ignore
         )
 
     return Settings().DAILY_LIMIT_PER_HOST - count_daily_request_per_host

bases/ecoindex/cli/app.py

@@ -138,30 +138,32 @@ def analyze(
             secho(f"⏲️ Crawling root url {url[0]} -> Wait a minute!", fg=colors.MAGENTA)
             with spinner():
                 urls = get_urls_recursive(main_url=url[0])
-                urls = urls if urls else url
+                urls = urls if urls else url  # type: ignore
 
             (
                 file_prefix,
                 input_file,
                 logger_file,
-            ) = get_file_prefix_input_file_logger_file(urls=urls)
+            ) = get_file_prefix_input_file_logger_file(urls=urls)  # type: ignore
 
         elif url:
-            urls = get_url_from_args(urls_arg=url)
+            urls = get_url_from_args(urls_arg=url)  # type: ignore
             (
                 file_prefix,
                 input_file,
                 logger_file,
-            ) = get_file_prefix_input_file_logger_file(urls=urls, tmp_folder=tmp_folder)
+            ) = get_file_prefix_input_file_logger_file(urls=urls, tmp_folder=tmp_folder)  # type: ignore
 
         elif urls_file:
-            urls = get_urls_from_file(urls_file=urls_file)
+            urls = get_urls_from_file(urls_file=urls_file)  # type: ignore
             (
                 file_prefix,
                 input_file,
                 logger_file,
             ) = get_file_prefix_input_file_logger_file(
-                urls=urls, urls_file=urls_file, tmp_folder=tmp_folder
+                urls=urls,  # type: ignore
+                urls_file=urls_file,
+                tmp_folder=tmp_folder,
             )
         elif sitemap:
             secho(
@@ -172,14 +174,14 @@ def analyze(
                 file_prefix,
                 input_file,
                 logger_file,
-            ) = get_file_prefix_input_file_logger_file(urls=urls)
+            ) = get_file_prefix_input_file_logger_file(urls=urls)  # type: ignore
 
         else:
             secho("🔥 You must provide an url...", fg=colors.RED)
             raise Exit(code=1)
 
         if input_file:
-            write_urls_to_file(file_prefix=file_prefix, urls=urls)
+            write_urls_to_file(file_prefix=file_prefix, urls=urls)  # type: ignore
             secho(f"📁️ Urls recorded in file `{input_file}`")
 
         if logger_file:
@@ -266,13 +268,13 @@ def analyze(
 
     Path(output_folder).mkdir(parents=True, exist_ok=True)
     write_results_to_file(
-        filename=output_filename, results=results, export_format=export_format
+        filename=str(output_filename), results=results, export_format=export_format
     )
     secho(f"🙌️ File {output_filename} written !", fg=colors.GREEN)
     if html_report:
         Report(
             results_file=output_filename,
-            output_path=output_folder,
+            output_path=str(output_folder),
             domain=file_prefix,
             date=time_now,
             language=html_report_language,
@@ -317,7 +319,7 @@ def report(
     output_folder = output_folder if output_folder else dirname(results_file)
 
     Report(
-        results_file=results_file,
+        results_file=Path(results_file),
         output_path=output_folder,
         domain=domain,
         date=datetime.now(),