From 5081752d242dfdd465e08c4bf7548fcd9e021b4b Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Thu, 24 Apr 2025 09:51:52 +0000 Subject: [PATCH 1/5] chore: Set UA to desktop / linux / chrome and add the ability to set custom headers in scraper --- bases/ecoindex/backend/routers/tasks.py | 24 ++++++++++++++++++++---- bases/ecoindex/worker/tasks.py | 10 ++++++++-- components/ecoindex/scraper/scrap.py | 15 +++++++++++++++ tasks/PoetryTaskfile.yml | 1 - 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index bf00a6a..2abcc6f 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -17,6 +17,7 @@ example_host_unreachable, ) from ecoindex.models.tasks import QueueTaskApi, QueueTaskApiBatch, QueueTaskResult +from ecoindex.scraper.scrap import EcoindexScraper from ecoindex.worker.tasks import ecoindex_batch_import_task, ecoindex_task from ecoindex.worker_component import app as task_app from fastapi import APIRouter, Depends, HTTPException, Response, status @@ -49,6 +50,13 @@ async def add_ecoindex_analysis_task( example=WebPage(url="https://www.ecoindex.fr", width=1920, height=1080), ), ], + custom_headers: Annotated[ + dict[str, str], + Body( + description="Custom headers to add to the request", + example={"X-My-Custom-Header": "MyValue"}, + ), + ] = {}, session: AsyncSession = Depends(get_session), ) -> str: if Settings().DAILY_LIMIT_PER_HOST: @@ -68,20 +76,28 @@ async def add_ecoindex_analysis_task( detail="This host is excluded from the analysis", ) + ua = EcoindexScraper.get_user_agent() + try: - ua = ua_generator.generate() - r = requests.head(url=web_page.url, timeout=5, headers=ua.headers.get()) + r = requests.head( + url=web_page.url, + timeout=5, + headers={**custom_headers, **ua.headers.get()}, + ) r.raise_for_status() except requests.exceptions.RequestException as e: raise HTTPException( status_code=e.response.status_code if e.response else status.HTTP_400_BAD_REQUEST, - detail=f"The URL {web_page.url} is unreachable. Are you really sure of this url? 🤔", + detail=f"The URL {web_page.url} is unreachable. Are you really sure of this url? 🤔 ({e.response.status_code if e.response else ''})", ) task_result = ecoindex_task.delay( # type: ignore - url=str(web_page.url), width=web_page.width, height=web_page.height + url=str(web_page.url), + width=web_page.width, + height=web_page.height, + custom_headers=custom_headers, ) return task_result.id diff --git a/bases/ecoindex/worker/tasks.py b/bases/ecoindex/worker/tasks.py index 339a53b..a90431a 100644 --- a/bases/ecoindex/worker/tasks.py +++ b/bases/ecoindex/worker/tasks.py @@ -37,9 +37,13 @@ queue="ecoindex", dont_autoretry_for=[EcoindexScraperStatusException, TypeError], ) -def ecoindex_task(self, url: str, width: int, height: int) -> str: +def ecoindex_task( + self, url: str, width: int, height: int, custom_headers: dict[str, str] +) -> str: queue_task_result = run( - async_ecoindex_task(self, url=url, width=width, height=height) + async_ecoindex_task( + self, url=url, width=width, height=height, custom_headers=custom_headers + ) ) return queue_task_result.model_dump_json() @@ -50,6 +54,7 @@ async def async_ecoindex_task( url: str, width: int, height: int, + custom_headers: dict[str, str], ) -> QueueTaskResult: try: session_generator = get_session() @@ -69,6 +74,7 @@ async def async_ecoindex_task( else None, screenshot_gid=Settings().SCREENSHOTS_GID, screenshot_uid=Settings().SCREENSHOTS_UID, + custom_headers=custom_headers, ).get_page_analysis() db_result = await save_ecoindex_result_db( diff --git a/components/ecoindex/scraper/scrap.py b/components/ecoindex/scraper/scrap.py index d8aa00c..90b5422 100644 --- a/components/ecoindex/scraper/scrap.py +++ b/components/ecoindex/scraper/scrap.py @@ -4,6 +4,9 @@ from time import sleep from uuid import uuid4 +from ua_generator.user_agent import UserAgent +from ua_generator import generate as ua_generate + from ecoindex.compute import compute_ecoindex from ecoindex.exceptions.scraper import EcoindexScraperStatusException from ecoindex.models.compute import PageMetrics, Result, ScreenShot, WindowSize @@ -28,6 +31,7 @@ def __init__( headless: bool = True, basic_auth: str | None = None, cookies: list[SetCookieParam] = [], + custom_headers: dict[str, str] = {}, ): self.url = url self.window_size = window_size @@ -45,6 +49,15 @@ def __init__( self.headless = headless self.basic_auth = basic_auth self.cookies = cookies + self.custom_headers = custom_headers + + @staticmethod + def get_user_agent() -> UserAgent: + return ua_generate( + device="desktop", + browser="chrome", + platform="linux", + ) @deprecated("This method is useless with new version of EcoindexScraper") def init_chromedriver(self): @@ -86,6 +99,8 @@ async def scrap_page(self) -> PageMetrics: } if self.basic_auth else None, + user_agent=self.get_user_agent().text, + extra_http_headers=self.custom_headers, ) await self.context.add_cookies(self.cookies) self.page = await self.context.new_page() diff --git a/tasks/PoetryTaskfile.yml b/tasks/PoetryTaskfile.yml index 7d40aaa..c498272 100644 --- a/tasks/PoetryTaskfile.yml +++ b/tasks/PoetryTaskfile.yml @@ -63,5 +63,4 @@ tasks: cmds: - poetry run playwright install chromium --with-deps silent: true - internal: true interactive: true From aa30e7aff9266d1d70b5e950b2ac263220153605 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Thu, 24 Apr 2025 09:57:41 +0000 Subject: [PATCH 2/5] chore: Add ua-generator to scraper --- poetry.lock | 49 +++++++++++------------- projects/ecoindex_api/poetry.lock | 13 +------ projects/ecoindex_api/pyproject.toml | 1 - projects/ecoindex_scraper/poetry.lock | 13 ++++++- projects/ecoindex_scraper/pyproject.toml | 1 + 5 files changed, 37 insertions(+), 40 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9bda58e..021d47e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -209,34 +209,31 @@ files = [ [[package]] name = "caio" -version = "0.9.22" +version = "0.9.24" description = "Asynchronous file IO for Linux MacOS or Windows." optional = false -python-versions = "<4,>=3.7" +python-versions = ">=3.9" files = [ - {file = "caio-0.9.22-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:979bef84869822a0b1d10c99f7240e2ca8b00c138a54bec1fcbef1163a6bc976"}, - {file = "caio-0.9.22-cp310-cp310-manylinux_2_34_aarch64.whl", hash = "sha256:bef2533b1444ce80df47ecce25ad8def6eb76a5ba8c1457074a16f5ab6e12670"}, - {file = "caio-0.9.22-cp310-cp310-manylinux_2_34_x86_64.whl", hash = "sha256:76fe5d98ff55099ec61a1863eed53ef353ea815e529c19a284865af73b3c84a3"}, - {file = "caio-0.9.22-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:dd99c28a85809d0954211d637dd0e2e5c0e385dcdfbdbc4ed914b549d7e0fb69"}, - {file = "caio-0.9.22-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:6ee67c4d63b77b3d07715c41939b71fee1bbb986219e398f3420ac31b43e65a8"}, - {file = "caio-0.9.22-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:fb335891181107bdb8f96dc133fc8e2612b62cd270219333659d6d53f75b1770"}, - {file = "caio-0.9.22-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:25aae4c3b846eeca63c7d75d63b3fd8deaba9650ef1168d8a239e6a927299ded"}, - {file = "caio-0.9.22-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:ac332b8e2c8e0840fe10ee6971b38f8dac6ad64ecf6087ee3f70cd376f511699"}, - {file = "caio-0.9.22-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:6f991e4812fd2d6e87f91ff78fcc7d4f299bd98765334756580d4ea42cad89f1"}, - {file = "caio-0.9.22-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c3d29f0f2f3a577b58794960dbfc78ef124a8b3e238d653a4c11c57f20651e34"}, - {file = "caio-0.9.22-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:a3307e064f423675c4ffd2608ecc86bc47652ec1547dab5f94b72d9599c23201"}, - {file = "caio-0.9.22-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:160a1dad60bbb4268811f88f817205fab6cba8fb50801fdd3d865b69999ea924"}, - {file = "caio-0.9.22-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:ceb32307d6d3cfa512220b53b578833ebd27f8ed0b534f15e3e88e6d598bffe7"}, - {file = "caio-0.9.22-cp38-cp38-manylinux_2_34_aarch64.whl", hash = "sha256:7145f2c47233af12cc0194a7bfe584c632b16fa00d3f900806eda527f89ce93b"}, - {file = "caio-0.9.22-cp38-cp38-manylinux_2_34_x86_64.whl", hash = "sha256:3bedd3e8ccc485cb20d4d56b0af1471e8f8b5ca3f0b5f1f21ebd087bfdcd21a7"}, - {file = "caio-0.9.22-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ae19a37b7294dc15c8775ec657464a4370d1f8351d0fc3a284a9f014514b9be5"}, - {file = "caio-0.9.22-cp39-cp39-manylinux_2_34_aarch64.whl", hash = "sha256:70e6a12c9d1db8fb7a07c5193ca352df1488787f1cf1c3814879dec754135031"}, - {file = "caio-0.9.22-cp39-cp39-manylinux_2_34_x86_64.whl", hash = "sha256:3f2f7f22c60844165359a285c5d38ca0d4ab10ca6f6def79abf6243d019df64b"}, - {file = "caio-0.9.22.tar.gz", hash = "sha256:7ea533d90e5fa0bba33bc8f4805b4c90f19e8d8ac5139a8033b92a6ab5c4012b"}, + {file = "caio-0.9.24-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d80322126a97ba572412b17b2f086ff95195de2c4261deb19db6bfcdc9ef7540"}, + {file = "caio-0.9.24-cp310-cp310-manylinux_2_34_aarch64.whl", hash = "sha256:37bc172349686139e8dc97fff7662c67b1837e18a67b99e8ef25585f2893d013"}, + {file = "caio-0.9.24-cp310-cp310-manylinux_2_34_x86_64.whl", hash = "sha256:ad7f0902bf952237e120606252c14ab3cb05995c9f79f39154b5248744864832"}, + {file = "caio-0.9.24-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:925b9e3748ce1a79386dfb921c0aee450e43225534551abd1398b1c08f9ba29f"}, + {file = "caio-0.9.24-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:3b4dc0a8fb9a58ab40f967ad5a8a858cc0bfb2348a580b4142595849457f9c9a"}, + {file = "caio-0.9.24-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:fa74d111b3b165bfad2e333367976bdf118bcf505a1cb44d3bcddea2849e3297"}, + {file = "caio-0.9.24-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7ae3566228383175265a7583107f21a7cb044a752ea29ba84fce7c1a49a05903"}, + {file = "caio-0.9.24-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:a306b0dda91cb4ca3170f066c114597f8ea41b3da578574a9d2b54f86963de68"}, + {file = "caio-0.9.24-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:8ee158e56128d865fb7d57a9c9c22fca4e8aa8d8664859c977a36fff3ccb3609"}, + {file = "caio-0.9.24-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1d47ef8d76aca74c17cb07339a441c5530fc4b8dd9222dfb1e1abd7f9f9b814f"}, + {file = "caio-0.9.24-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:d15fc746c4bf0077d75df05939d1e97c07ccaa8e580681a77021d6929f65d9f4"}, + {file = "caio-0.9.24-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:9368eae0a9badd5f31264896c51b47431d96c0d46f1979018fb1d20c49f56156"}, + {file = "caio-0.9.24-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3f0e5a645ef4e7bb7a81e10ae2a7aef14988cb2cb4354588c6bf6f6f3f6de72a"}, + {file = "caio-0.9.24-cp39-cp39-manylinux_2_34_aarch64.whl", hash = "sha256:08304fa80af7771c78a5bcc923449c7ec8134d589b50d48c66320f85552c7ae2"}, + {file = "caio-0.9.24-cp39-cp39-manylinux_2_34_x86_64.whl", hash = "sha256:5339ced0764e10242a50ccb21db7f0d9c359881db0f72fa2c5e45ed828ffacf7"}, + {file = "caio-0.9.24.tar.gz", hash = "sha256:5bcdecaea02a9aa8e3acf0364eff8ad9903d57d70cdb274a42270126290a77f1"}, ] [package.extras] -develop = ["aiomisc-pytest", "pytest", "pytest-cov"] +develop = ["aiomisc-pytest", "coveralls", "pylama[toml]", "pytest", "pytest-cov", "setuptools"] [[package]] name = "celery" @@ -1024,13 +1021,13 @@ test = ["objgraph", "psutil"] [[package]] name = "h11" -version = "0.14.0" +version = "0.16.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, - {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, ] [[package]] diff --git a/projects/ecoindex_api/poetry.lock b/projects/ecoindex_api/poetry.lock index be5dd50..3abb620 100644 --- a/projects/ecoindex_api/poetry.lock +++ b/projects/ecoindex_api/poetry.lock @@ -1519,17 +1519,6 @@ files = [ {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] -[[package]] -name = "ua-generator" -version = "2.0.5" -description = "A random user-agent generator" -optional = false -python-versions = ">=3.9" -files = [ - {file = "ua_generator-2.0.5-py3-none-any.whl", hash = "sha256:ed7437db60ad1c9a3dfa89e32700a37636944fd7adfccd12013dfbb3b540116f"}, - {file = "ua_generator-2.0.5.tar.gz", hash = "sha256:5a75d25d71d574da8ec259c06ad55009c8c91cd4b520c2820ad9ba9d6dd28053"}, -] - [[package]] name = "urllib3" version = "2.4.0" @@ -1633,4 +1622,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "3288f9f522873c4bdde13c1da112a7bc401ce3d6a712754102ab99d381d7470b" +content-hash = "a1d61d1334f4cb9c3cafe5a11965eee92ee53c708f18b5a40360225022adc3f3" diff --git a/projects/ecoindex_api/pyproject.toml b/projects/ecoindex_api/pyproject.toml index 32e9243..bc37d19 100644 --- a/projects/ecoindex_api/pyproject.toml +++ b/projects/ecoindex_api/pyproject.toml @@ -43,7 +43,6 @@ cryptography = "^44.0.2" [tool.poetry.group.backend.dependencies] uvicorn = "^0.23.2" -ua-generator = "^2.0.3" [tool.poetry.group.worker.dependencies] pillow = "^10.3.0" diff --git a/projects/ecoindex_scraper/poetry.lock b/projects/ecoindex_scraper/poetry.lock index b6365a0..f0c9d4a 100644 --- a/projects/ecoindex_scraper/poetry.lock +++ b/projects/ecoindex_scraper/poetry.lock @@ -464,7 +464,18 @@ files = [ [package.dependencies] typing-extensions = ">=4.12.0" +[[package]] +name = "ua-generator" +version = "2.0.5" +description = "A random user-agent generator" +optional = false +python-versions = ">=3.9" +files = [ + {file = "ua_generator-2.0.5-py3-none-any.whl", hash = "sha256:ed7437db60ad1c9a3dfa89e32700a37636944fd7adfccd12013dfbb3b540116f"}, + {file = "ua_generator-2.0.5.tar.gz", hash = "sha256:5a75d25d71d574da8ec259c06ad55009c8c91cd4b520c2820ad9ba9d6dd28053"}, +] + [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "079de5cdf705cd640f1ccb7dc3fb827d76c2d7a18e72564b7869f97d33006f9b" +content-hash = "a6c6f228991742415049c99c6b44c2ef3994e544afc50baa4ff72265bc875578" diff --git a/projects/ecoindex_scraper/pyproject.toml b/projects/ecoindex_scraper/pyproject.toml index 41a063d..d61743d 100644 --- a/projects/ecoindex_scraper/pyproject.toml +++ b/projects/ecoindex_scraper/pyproject.toml @@ -26,6 +26,7 @@ typing-extensions = "^4.8.0" pyyaml = "^6.0.1" pillow = "^10.1.0" setuptools = ">=69.5.1,<71.0.0" +ua-generator = "^2.0.5" [build-system] requires = ["poetry-core>=1.0.0"] From b0512fb8f7f599c396649d6323cbb3ca86b28e21 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Thu, 24 Apr 2025 10:05:36 +0000 Subject: [PATCH 3/5] chore: Use custom headers to set user agent API --- bases/ecoindex/backend/routers/tasks.py | 5 +++-- components/ecoindex/scraper/scrap.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index 2abcc6f..2c17ea9 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -77,12 +77,13 @@ async def add_ecoindex_analysis_task( ) ua = EcoindexScraper.get_user_agent() + headers = {**custom_headers, **ua.headers.get()} try: r = requests.head( url=web_page.url, timeout=5, - headers={**custom_headers, **ua.headers.get()}, + headers=headers, ) r.raise_for_status() except requests.exceptions.RequestException as e: @@ -97,7 +98,7 @@ async def add_ecoindex_analysis_task( url=str(web_page.url), width=web_page.width, height=web_page.height, - custom_headers=custom_headers, + custom_headers=headers, ) return task_result.id diff --git a/components/ecoindex/scraper/scrap.py b/components/ecoindex/scraper/scrap.py index 90b5422..b2e7786 100644 --- a/components/ecoindex/scraper/scrap.py +++ b/components/ecoindex/scraper/scrap.py @@ -99,7 +99,6 @@ async def scrap_page(self) -> PageMetrics: } if self.basic_auth else None, - user_agent=self.get_user_agent().text, extra_http_headers=self.custom_headers, ) await self.context.add_cookies(self.cookies) From 7b5a07816828bc6714170ddab87d3f29c7f428fe Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Thu, 24 Apr 2025 10:11:18 +0000 Subject: [PATCH 4/5] chore: add ua-generator to API and CLI --- projects/ecoindex_api/poetry.lock | 13 ++++++++++++- projects/ecoindex_api/pyproject.toml | 1 + projects/ecoindex_cli/poetry.lock | 13 ++++++++++++- projects/ecoindex_cli/pyproject.toml | 1 + 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/projects/ecoindex_api/poetry.lock b/projects/ecoindex_api/poetry.lock index 3abb620..d1765a2 100644 --- a/projects/ecoindex_api/poetry.lock +++ b/projects/ecoindex_api/poetry.lock @@ -1519,6 +1519,17 @@ files = [ {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] +[[package]] +name = "ua-generator" +version = "2.0.5" +description = "A random user-agent generator" +optional = false +python-versions = ">=3.9" +files = [ + {file = "ua_generator-2.0.5-py3-none-any.whl", hash = "sha256:ed7437db60ad1c9a3dfa89e32700a37636944fd7adfccd12013dfbb3b540116f"}, + {file = "ua_generator-2.0.5.tar.gz", hash = "sha256:5a75d25d71d574da8ec259c06ad55009c8c91cd4b520c2820ad9ba9d6dd28053"}, +] + [[package]] name = "urllib3" version = "2.4.0" @@ -1622,4 +1633,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "a1d61d1334f4cb9c3cafe5a11965eee92ee53c708f18b5a40360225022adc3f3" +content-hash = "6720b26abf260c07f3ee660ce629279dc87cfe83da6c4b8271cd9035830a04dd" diff --git a/projects/ecoindex_api/pyproject.toml b/projects/ecoindex_api/pyproject.toml index bc37d19..8f191db 100644 --- a/projects/ecoindex_api/pyproject.toml +++ b/projects/ecoindex_api/pyproject.toml @@ -40,6 +40,7 @@ sqlmodel = "^0.0.14" sentry-sdk = "^2.8.0" setuptools = "^75.6.0" cryptography = "^44.0.2" +ua-generator = "^2.0.5" [tool.poetry.group.backend.dependencies] uvicorn = "^0.23.2" diff --git a/projects/ecoindex_cli/poetry.lock b/projects/ecoindex_cli/poetry.lock index 949b01e..40b4d6e 100644 --- a/projects/ecoindex_cli/poetry.lock +++ b/projects/ecoindex_cli/poetry.lock @@ -2188,6 +2188,17 @@ files = [ {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] +[[package]] +name = "ua-generator" +version = "2.0.5" +description = "A random user-agent generator" +optional = false +python-versions = ">=3.9" +files = [ + {file = "ua_generator-2.0.5-py3-none-any.whl", hash = "sha256:ed7437db60ad1c9a3dfa89e32700a37636944fd7adfccd12013dfbb3b540116f"}, + {file = "ua_generator-2.0.5.tar.gz", hash = "sha256:5a75d25d71d574da8ec259c06ad55009c8c91cd4b520c2820ad9ba9d6dd28053"}, +] + [[package]] name = "urllib3" version = "2.4.0" @@ -2287,4 +2298,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "bc406663a9e8852446f94e76f2d0a3dc2eab1441b7ff99d54aed03153f952c23" +content-hash = "9a3ba3ab6511cbd8b11937ba7fed39fe3d8206ecc3a1aeee8a526fd122b79723" diff --git a/projects/ecoindex_cli/pyproject.toml b/projects/ecoindex_cli/pyproject.toml index c3727f3..a8a22e3 100644 --- a/projects/ecoindex_cli/pyproject.toml +++ b/projects/ecoindex_cli/pyproject.toml @@ -35,6 +35,7 @@ pyyaml = "^6.0.1" rich = "^13.6.0" scrapy = "^2.11.0" typer = "^0.9.0" +ua-generator = "^2.0.5" [tool.poetry.scripts] ecoindex-cli = "ecoindex.cli.app:app" From c207dba5f0ccc0d2b13130faae52ab77bbe7ccef Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Thu, 24 Apr 2025 10:15:25 +0000 Subject: [PATCH 5/5] chore: remove unused import --- bases/ecoindex/backend/routers/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index 2c17ea9..e4885f8 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -3,7 +3,6 @@ import requests from celery.result import AsyncResult -import ua_generator from ecoindex.backend.dependencies.validation import validate_api_key_batch from ecoindex.backend.models.dependencies_parameters.id import IdParameter from ecoindex.backend.utils import check_quota