From c809b97648a46ba6e6c6319c1291937e3e283abe Mon Sep 17 00:00:00 2001 From: JE-Chen <33644111+JE-Chen@users.noreply.github.com> Date: Sat, 29 Mar 2025 03:38:13 +0800 Subject: [PATCH 1/4] Update stable and dev version Update stable and dev version --- dev.toml | 6 ++---- pyproject.toml | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/dev.toml b/dev.toml index 1c0f0e1..27a3159 100644 --- a/dev.toml +++ b/dev.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "automation_file_dev" -version = "0.0.25" +version = "0.0.26" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] @@ -17,8 +17,7 @@ license = { text = "MIT" } dependencies = [ "google-api-python-client", "google-auth-httplib2", - "google-auth-oauthlib", - "APScheduler" + "google-auth-oauthlib" ] classifiers = [ "Programming Language :: Python :: 3.9", @@ -26,7 +25,6 @@ classifiers = [ "Environment :: Win32 (MS Windows)", "Environment :: MacOS X", "Environment :: X11 Applications", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent" ] diff --git a/pyproject.toml b/pyproject.toml index 94a2b80..b42a61a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "automation_file" -version = "0.0.23" +version = "0.0.24" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] @@ -18,7 +18,6 @@ dependencies = [ "google-api-python-client", "google-auth-httplib2", "google-auth-oauthlib", - "APScheduler" ] classifiers = [ "Programming Language :: Python :: 3.9", From 66c6801a5376c3e0ce35ded0b3b2d8c5dce7f048 Mon Sep 17 00:00:00 2001 From: JE-Chen <33644111+JE-Chen@users.noreply.github.com> Date: Mon, 7 Apr 2025 23:34:31 +0800 Subject: [PATCH 2/4] Update project toml Update project toml --- dev.toml | 2 +- pyproject.toml | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dev.toml b/dev.toml index 27a3159..24c5009 100644 --- a/dev.toml +++ b/dev.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "automation_file_dev" -version = "0.0.26" +version = "0.0.27" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] diff --git a/pyproject.toml b/pyproject.toml index b42a61a..7dd7b75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,14 +6,14 @@ build-backend = "setuptools.build_meta" [project] name = "automation_file" -version = "0.0.24" +version = "0.0.25" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] description = "" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9" -license = { text = "MIT" } +license-files = ["LICENSE"] dependencies = [ "google-api-python-client", "google-auth-httplib2", @@ -25,7 +25,6 @@ classifiers = [ "Environment :: Win32 (MS Windows)", "Environment :: MacOS X", "Environment :: X11 Applications", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent" ] From 2090c4854163fdc915972f3e68baf34ee386badd Mon Sep 17 00:00:00 2001 From: JE-Chen <33644111+JE-Chen@users.noreply.github.com> Date: Sun, 13 Apr 2025 18:24:34 +0800 Subject: [PATCH 3/4] Add download file and update dev version Add download file and update dev version --- automation_file/__init__.py | 4 +- .../scheduler => remote/download}/__init__.py | 0 automation_file/remote/download/file.py | 35 +++ .../utils/executor/action_executor.py | 20 -- .../utils/scheduler/extend_apscheduler.py | 215 ------------------ pyproject.toml | 8 +- requirements.txt | 5 +- dev.toml => stable.toml | 10 +- 8 files changed, 53 insertions(+), 244 deletions(-) rename automation_file/{utils/scheduler => remote/download}/__init__.py (100%) create mode 100644 automation_file/remote/download/file.py delete mode 100644 automation_file/utils/scheduler/extend_apscheduler.py rename dev.toml => stable.toml (86%) diff --git a/automation_file/__init__.py b/automation_file/__init__.py index a97bd38..033b2f2 100644 --- a/automation_file/__init__.py +++ b/automation_file/__init__.py @@ -18,6 +18,7 @@ from automation_file.utils.file_process.get_dir_file_list import get_dir_files_as_list from automation_file.utils.json.json_file import read_action_json from automation_file.utils.project.create_project_structure import create_project_dir +from automation_file.remote.download.file import download_file __all__ = [ "copy_file", "rename_file", "remove_file", "copy_all_file_to_dir", "copy_specify_extension_file", @@ -27,5 +28,6 @@ "drive_upload_dir_to_folder", "drive_upload_to_folder", "drive_upload_dir_to_drive", "drive_upload_to_drive", "drive_add_folder", "drive_share_file_to_anyone", "drive_share_file_to_domain", "drive_share_file_to_user", "drive_delete_file", "drive_download_file", "drive_download_file_from_folder", "execute_action", "execute_files", - "add_command_to_executor", "read_action_json", "get_dir_files_as_list", "create_project_dir" + "add_command_to_executor", "read_action_json", "get_dir_files_as_list", "create_project_dir", + "download_file" ] diff --git a/automation_file/utils/scheduler/__init__.py b/automation_file/remote/download/__init__.py similarity index 100% rename from automation_file/utils/scheduler/__init__.py rename to automation_file/remote/download/__init__.py diff --git a/automation_file/remote/download/file.py b/automation_file/remote/download/file.py new file mode 100644 index 0000000..a0e0d3c --- /dev/null +++ b/automation_file/remote/download/file.py @@ -0,0 +1,35 @@ +import requests +from tqdm import tqdm + +from automation_file.utils.logging.loggin_instance import file_automation_logger + + +def download_file(file_url: str, file_name: str, chunk_size: int = 1024, timeout: int = 10): + try: + response = requests.get(file_url, stream=True, timeout=10) + response.raise_for_status() + total_size = int(response.headers.get('content-length', 0)) + with open(file_name, 'wb') as file: + if total_size > 0: + with tqdm( + total=total_size, unit='B', unit_scale=True, desc=file_name + ) as progress: + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: + file.write(chunk) + progress.update(len(chunk)) + else: + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: + file.write(chunk) + + file_automation_logger.info(f"File download is complete. Saved as: {file_name}") + except requests.exceptions.HTTPError as http_err: + file_automation_logger.error(f"HTTP error：{http_err}") + except requests.exceptions.ConnectionError: + file_automation_logger.error("Connection error. Please check your internet connection.") + except requests.exceptions.Timeout: + file_automation_logger.error("Request timed out. The server did not respond.") + except Exception as err: + file_automation_logger.error(f"Error：{err}") + diff --git a/automation_file/utils/executor/action_executor.py b/automation_file/utils/executor/action_executor.py index f8f7e66..28b74ec 100644 --- a/automation_file/utils/executor/action_executor.py +++ b/automation_file/utils/executor/action_executor.py @@ -25,7 +25,6 @@ from automation_file.utils.json.json_file import read_action_json from automation_file.utils.logging.loggin_instance import file_automation_logger from automation_file.utils.package_manager.package_manager_class import package_manager -from automation_file.utils.scheduler.extend_apscheduler import scheduler_manager class Executor(object): @@ -72,15 +71,6 @@ def __init__(self): "FA_execute_action": self.execute_action, "FA_execute_files": self.execute_files, "FA_add_package_to_executor": package_manager.add_package_to_executor, - # Scheduler - "FA_scheduler_event_trigger": self.scheduler_event_trigger, - "FA_remove_blocking_scheduler_job": scheduler_manager.remove_blocking_job, - "FA_remove_nonblocking_scheduler_job": scheduler_manager.remove_nonblocking_job, - "FA_start_blocking_scheduler": scheduler_manager.start_block_scheduler, - "FA_start_nonblocking_scheduler": scheduler_manager.start_nonblocking_scheduler, - "FA_start_all_scheduler": scheduler_manager.start_all_scheduler, - "FA_shutdown_blocking_scheduler": scheduler_manager.shutdown_blocking_scheduler, - "FA_shutdown_nonblocking_scheduler": scheduler_manager.shutdown_nonblocking_scheduler, } # get all builtin function and add to event dict for function in getmembers(builtins, isbuiltin): @@ -145,16 +135,6 @@ def execute_files(self, execute_files_list: list) -> list: execute_detail_list.append(self.execute_action(read_action_json(file))) return execute_detail_list - def scheduler_event_trigger( - self, function: str, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, scheduler_type: str = "nonblocking", wait_type: str = "secondly", - wait_value: int = 1, **trigger_args: Any) -> None: - if scheduler_type == "nonblocking": - scheduler_event = scheduler_manager.nonblocking_scheduler_event_dict.get(wait_type) - else: - scheduler_event = scheduler_manager.blocking_scheduler_event_dict.get(wait_type) - scheduler_event(self.event_dict.get(function), id, args, kwargs, wait_value, **trigger_args) - executor = Executor() package_manager.executor = executor diff --git a/automation_file/utils/scheduler/extend_apscheduler.py b/automation_file/utils/scheduler/extend_apscheduler.py deleted file mode 100644 index 36abd01..0000000 --- a/automation_file/utils/scheduler/extend_apscheduler.py +++ /dev/null @@ -1,215 +0,0 @@ -from datetime import datetime -from typing import Callable, Any, Union - -from apscheduler.job import Job -from apscheduler.schedulers.background import BackgroundScheduler -from apscheduler.schedulers.blocking import BlockingScheduler -from apscheduler.util import undefined - - -class SchedulerManager(object): - - def __init__(self): - self._blocking_schedulers: BlockingScheduler = BlockingScheduler() - self._background_schedulers: BackgroundScheduler = BackgroundScheduler() - self.blocking_scheduler_event_dict = { - "secondly": self.add_interval_blocking_secondly, - "minutely": self.add_interval_blocking_minutely, - "hourly": self.add_interval_blocking_hourly, - "daily": self.add_interval_blocking_daily, - "weekly": self.add_interval_blocking_weekly, - } - self.nonblocking_scheduler_event_dict = { - "secondly": self.add_interval_nonblocking_secondly, - "minutely": self.add_interval_nonblocking_minutely, - "hourly": self.add_interval_nonblocking_hourly, - "daily": self.add_interval_nonblocking_daily, - "weekly": self.add_interval_nonblocking_weekly, - } - - def add_blocking_job( - self, func: Callable, trigger: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, id: str = None, name: str = None, - misfire_grace_time: int = undefined, coalesce: bool = undefined, max_instances: int = undefined, - next_run_time: datetime = undefined, jobstore: str = 'default', executor: str = 'default', - replace_existing: bool = False, **trigger_args: Any) -> Job: - """ - Just an apscheduler add job wrapper. - :param func: callable (or a textual reference to one) to run at the given time - :param str|apscheduler.triggers.base.BaseTrigger trigger: trigger that determines when - ``func`` is called - :param list|tuple args: list of positional arguments to call func with - :param dict kwargs: dict of keyword arguments to call func with - :param str|unicode id: explicit identifier for the job (for modifying it later) - :param str|unicode name: textual description of the job - :param int misfire_grace_time: seconds after the designated runtime that the job is still - allowed to be run (or ``None`` to allow the job to run no matter how late it is) - :param bool coalesce: run once instead of many times if the scheduler determines that the - job should be run more than once in succession - :param int max_instances: maximum number of concurrently running instances allowed for this - job - :param datetime next_run_time: when to first run the job, regardless of the trigger (pass - ``None`` to add the job as paused) - :param str|unicode jobstore: alias of the job store to store the job in - :param str|unicode executor: alias of the executor to run the job with - :param bool replace_existing: ``True`` to replace an existing job with the same ``id`` - (but retain the number of runs from the existing one) - :return: Job - """ - params = locals() - params.pop("self") - params.pop("trigger_args") - return self._blocking_schedulers.add_job(**params, **trigger_args) - - def add_nonblocking_job( - self, func: Callable, trigger: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, id: str = None, name: str = None, - misfire_grace_time: int = undefined, coalesce: bool = undefined, max_instances: int = undefined, - next_run_time: datetime = undefined, jobstore: str = 'default', executor: str = 'default', - replace_existing: bool = False, **trigger_args: Any) -> Job: - """ - Just an apscheduler add job wrapper. - :param func: callable (or a textual reference to one) to run at the given time - :param str|apscheduler.triggers.base.BaseTrigger trigger: trigger that determines when - ``func`` is called - :param list|tuple args: list of positional arguments to call func with - :param dict kwargs: dict of keyword arguments to call func with - :param str|unicode id: explicit identifier for the job (for modifying it later) - :param str|unicode name: textual description of the job - :param int misfire_grace_time: seconds after the designated runtime that the job is still - allowed to be run (or ``None`` to allow the job to run no matter how late it is) - :param bool coalesce: run once instead of many times if the scheduler determines that the - job should be run more than once in succession - :param int max_instances: maximum number of concurrently running instances allowed for this - job - :param datetime next_run_time: when to first run the job, regardless of the trigger (pass - ``None`` to add the job as paused) - :param str|unicode jobstore: alias of the job store to store the job in - :param str|unicode executor: alias of the executor to run the job with - :param bool replace_existing: ``True`` to replace an existing job with the same ``id`` - (but retain the number of runs from the existing one) - :return: Job - """ - params = locals() - params.pop("self") - params.pop("trigger_args") - return self._background_schedulers.add_job(**params, **trigger_args) - - def get_blocking_scheduler(self) -> BlockingScheduler: - """ - Return self blocking scheduler - :return: BlockingScheduler - """ - return self._blocking_schedulers - - def get_nonblocking_scheduler(self) -> BackgroundScheduler: - """ - Return self background scheduler - :return: BackgroundScheduler - """ - return self._background_schedulers - - def start_block_scheduler(self, *args: Any, **kwargs: Any) -> None: - """ - Start blocking scheduler - :return: None - """ - self._blocking_schedulers.start(*args, **kwargs) - - def start_nonblocking_scheduler(self, *args: Any, **kwargs: Any) -> None: - """ - Start background scheduler - :return: None - """ - self._background_schedulers.start(*args, **kwargs) - - def start_all_scheduler(self, *args: Any, **kwargs: Any) -> None: - """ - Start background and blocking scheduler - :return: None - """ - self._blocking_schedulers.start(*args, **kwargs) - self._background_schedulers.start(*args, **kwargs) - - def add_interval_blocking_secondly( - self, function: Callable, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, seconds: int = 1, **trigger_args: Any) -> Job: - return self.add_blocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, seconds=seconds, **trigger_args) - - def add_interval_blocking_minutely( - self, function: Callable, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, minutes: int = 1, **trigger_args: Any) -> Job: - return self.add_blocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, minutes=minutes, **trigger_args) - - def add_interval_blocking_hourly( - self, function: Callable, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, hours: int = 1, **trigger_args: Any) -> Job: - return self.add_blocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, hours=hours, **trigger_args) - - def add_interval_blocking_daily( - self, function: Callable, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, days: int = 1, **trigger_args: Any) -> Job: - return self.add_blocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, days=days, **trigger_args) - - def add_interval_blocking_weekly( - self, function: Callable, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, weeks: int = 1, **trigger_args: Any) -> Job: - return self.add_blocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, weeks=weeks, **trigger_args) - - def add_interval_nonblocking_secondly( - self, function: Callable, id: str = None, args: list = None, - kwargs: dict = None, seconds: int = 1, **trigger_args: Any) -> Job: - return self.add_nonblocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, seconds=seconds, **trigger_args) - - def add_interval_nonblocking_minutely( - self, function: Callable, id: str = None, args: list = None, - kwargs: dict = None, minutes: int = 1, **trigger_args: Any) -> Job: - return self.add_nonblocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, minutes=minutes, **trigger_args) - - def add_interval_nonblocking_hourly( - self, function: Callable, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, hours: int = 1, **trigger_args: Any) -> Job: - return self.add_nonblocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, hours=hours, **trigger_args) - - def add_interval_nonblocking_daily( - self, function: Callable, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, days: int = 1, **trigger_args: Any) -> Job: - return self.add_nonblocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, days=days, **trigger_args) - - def add_interval_nonblocking_weekly( - self, function: Callable, id: str = None, args: Union[list, tuple] = None, - kwargs: dict = None, weeks: int = 1, **trigger_args: Any) -> Job: - return self.add_nonblocking_job( - func=function, trigger="interval", id=id, args=args, kwargs=kwargs, weeks=weeks, **trigger_args) - - def add_cron_blocking( - self, function: Callable, id: str = None, **trigger_args: Any) -> Job: - return self.add_blocking_job(func=function, id=id, trigger="cron", **trigger_args) - - def add_cron_nonblocking( - self, function: Callable, id: str = None, **trigger_args: Any) -> Job: - return self.add_nonblocking_job(func=function, id=id, trigger="cron", **trigger_args) - - def remove_blocking_job(self, id: str, jobstore: str = 'default') -> Any: - return self._blocking_schedulers.remove_job(job_id=id, jobstore=jobstore) - - def remove_nonblocking_job(self, id: str, jobstore: str = 'default') -> Any: - return self._background_schedulers.remove_job(job_id=id, jobstore=jobstore) - - def shutdown_blocking_scheduler(self, wait: bool = False) -> None: - self._blocking_schedulers.shutdown(wait=wait) - - def shutdown_nonblocking_scheduler(self, wait: bool = False) -> None: - self._background_schedulers.shutdown(wait=wait) - - -scheduler_manager = SchedulerManager() diff --git a/pyproject.toml b/pyproject.toml index 7dd7b75..cdcba83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,19 +5,21 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "automation_file" -version = "0.0.25" +name = "automation_file_dev" +version = "0.0.28" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] description = "" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9" -license-files = ["LICENSE"] +license = { text = "MIT" } dependencies = [ "google-api-python-client", "google-auth-httplib2", "google-auth-oauthlib", + "requests", + "tqdm" ] classifiers = [ "Programming Language :: Python :: 3.9", diff --git a/requirements.txt b/requirements.txt index b09bdbc..2cb4f68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,7 @@ automation_file google-api-python-client google-auth-httplib2 google-auth-oauthlib -APScheduler +PySide6 +requests +protobuf +tqdm \ No newline at end of file diff --git a/dev.toml b/stable.toml similarity index 86% rename from dev.toml rename to stable.toml index 24c5009..0639af3 100644 --- a/dev.toml +++ b/stable.toml @@ -5,19 +5,21 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "automation_file_dev" -version = "0.0.27" +name = "automation_file" +version = "0.0.25" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] description = "" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9" -license = { text = "MIT" } +license-files = ["LICENSE"] dependencies = [ "google-api-python-client", "google-auth-httplib2", - "google-auth-oauthlib" + "google-auth-oauthlib", + "requests", + "tqdm" ] classifiers = [ "Programming Language :: Python :: 3.9", From ce66926d748e140148bd41f38efbe27dd48b64ea Mon Sep 17 00:00:00 2001 From: JE-Chen <33644111+JE-Chen@users.noreply.github.com> Date: Sun, 13 Apr 2025 18:55:53 +0800 Subject: [PATCH 4/4] update stable version update stable version --- stable.toml => dev.toml | 6 +++--- pyproject.toml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) rename stable.toml => dev.toml (91%) diff --git a/stable.toml b/dev.toml similarity index 91% rename from stable.toml rename to dev.toml index 0639af3..cdcba83 100644 --- a/stable.toml +++ b/dev.toml @@ -5,15 +5,15 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "automation_file" -version = "0.0.25" +name = "automation_file_dev" +version = "0.0.28" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] description = "" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9" -license-files = ["LICENSE"] +license = { text = "MIT" } dependencies = [ "google-api-python-client", "google-auth-httplib2", diff --git a/pyproject.toml b/pyproject.toml index cdcba83..0134409 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,15 +5,15 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "automation_file_dev" -version = "0.0.28" +name = "automation_file" +version = "0.0.26" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] description = "" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9" -license = { text = "MIT" } +license-files = ["LICENSE"] dependencies = [ "google-api-python-client", "google-auth-httplib2",