Merge pull request #52 from OpenDataAlex/process_tracker_python-50

Process tracker python 50

Author: OpenDataAlex

.gitignore

@@ -173,4 +173,3 @@ pip-selfcheck.json
 .idea/misc.xml
 .idea/modules.xml
 .idea/process_tracker_python.iml
-/tests/test_process_tracker.py

dbscripts/mysql_process_tracker.sql

@@ -272,3 +272,31 @@ create index process_run_actor_id
 create index process_status_id
 	on process_tracking (process_status_id);
 
+create table process_tracker.cluster_tracking
+(
+	cluster_id int auto_increment
+		primary key,
+	cluster_name varchar(250) not null,
+	cluster_max_memory int null,
+	cluster_max_memory_unit char(2) null,
+	cluster_max_processing int null,
+	cluster_max_processing_unit varchar(3) null,
+	cluster_current_memory_usage int null,
+	cluster_current_process_usage int null,
+	constraint cluster_tracking_cluster_name_uindex
+		unique (cluster_name)
+)
+comment 'Capacity cluster tracking';
+
+create table process_tracker.cluster_process
+(
+	cluster_id int not null,
+	process_id int not null,
+	primary key (cluster_id, process_id),
+	constraint cluster_process_fk01
+		foreign key (cluster_id) references process_tracker.cluster_tracking (cluster_id),
+	constraint cluster_process_fk02
+		foreign key (process_id) references process_tracker.process (process_id)
+)
+comment 'Relationship tracking between processes and performance clusters.';
+

dbscripts/postgresql_process_tracker.sql

@@ -114,10 +114,10 @@ create table process
 			primary key,
 	process_name varchar(250) not null,
 	total_record_count integer default 0 not null,
-	process_type_id integer not null
+	process_type_id integer null
 		constraint process_fk02
 			references process_type_lkup,
-	process_tool_id integer not null
+	process_tool_id integer null
 		constraint process_fk03
 			references tool_lkup,
 	last_failed_run_date_time timestamp default '1900-01-01 00:00:00'::timestamp without time zone not null
@@ -417,3 +417,40 @@ comment on table extract_dependency is 'Table tracking interdependencies between
 
 alter table extract_dependency owner to pt_admin;
 
+create table cluster_tracking
+(
+	cluster_id serial not null
+		constraint cluster_tracking_pk
+			primary key,
+	cluster_name varchar(250) not null,
+	cluster_max_memory integer null,
+	cluster_max_memory_unit char(2) null,
+	cluster_max_processing integer null,
+	cluster_max_processing_unit varchar(3) null,
+	cluster_current_memory_usage integer,
+	cluster_current_process_usage integer
+);
+
+comment on table cluster_tracking is 'Capacity cluster tracking';
+
+alter table cluster_tracking owner to pt_admin;
+
+create unique index cluster_tracking_cluster_name_uindex
+	on cluster_tracking (cluster_name);
+
+create table cluster_process
+(
+	cluster_id integer not null
+		constraint cluster_process_fk01
+			references cluster_tracking,
+	process_id integer not null
+		constraint cluster_process_fk02
+			references process,
+	constraint cluster_process_pk
+		primary key (cluster_id, process_id)
+);
+
+comment on table cluster_process is 'Relationship tracking between processes and performance clusters.';
+
+alter table cluster_process owner to pt_admin;
+

process_tracker/cli.py

@@ -54,9 +54,39 @@ def upgrade():
     "-p", "--parent", help="The parent process' name, if creating a process dependency"
 )
 @click.option(
-    "-c", "--child", help="The child process' name, if creating a process dependency"
+    "-c",
+    "--child",
+    help="The child process' name, if creating a process dependency.  For cluster/process relationships, the name of "
+    "the process.",
 )
-def create(topic, name, parent=None, child=None):
+@click.option(
+    "--max-processing",
+    help="For clusters, the max processing ability allocated to the cluster",
+)
+@click.option(
+    "--max-memory", help="For clusters, the max memory allocated to the cluster"
+)
+@click.option(
+    "--processing-unit",
+    help="For clusters, the unit of processing ability (Ghz, CPU, DPU, etc.",
+)
+@click.option(
+    "--memory-unit", help="For clusters, the unit of allocated memory (MB, GB, etc."
+)
+@click.option(
+    "--cluster", help="For cluster/process relationships, the name of the cluster"
+)
+def create(
+    topic,
+    name,
+    parent=None,
+    child=None,
+    max_processing=None,
+    processing_unit=None,
+    max_memory=None,
+    memory_unit=None,
+    cluster=None,
+):
     """
     Create an item that is within the valid topics list.
     :param topic: The name of the topic.
@@ -65,11 +95,32 @@ def create(topic, name, parent=None, child=None):
     :type name: string
     :param parent: The parent process' name, if creating a process dependency
     :type parent: string
-    :param child: The child process' name, if creating a process dependency
+    :param child: The child process' name, if creating a process dependency.  For cluster/process relationships, the
+                  name of the process.
     :type child: string
+    :param max_processing: For performance clusters, the maximum processing ability allocated to the cluster
+    :type max_processing: integer
+    :param max_memory: For performance clusters, the maximum memory allocated to the cluster
+    :type max_memory: integer
+    :param processing_unit: For performance clusters, the unit of processing ability allocated to the cluster
+    :type processing_unit: string
+    :param memory_unit: For performance clusters, the unit of allocated memory to the cluster
+    :type memory_unit: string
+    :param cluster: For cluster/process relationships, the name of the cluster.
+    :type cluster: string
     """
     click.echo("Attempting to create %s with name %s" % (topic, name))
-    data_store.topic_creator(topic=topic, name=name, parent=parent, child=child)
+    data_store.topic_creator(
+        topic=topic,
+        name=name,
+        parent=parent,
+        child=child,
+        max_processing=max_processing,
+        max_memory=max_memory,
+        processing_unit=processing_unit,
+        memory_unit=memory_unit,
+        cluster=cluster,
+    )
 
 
 @main.command()
@@ -81,7 +132,10 @@ def create(topic, name, parent=None, child=None):
 @click.option(
     "-c", "--child", help="The child process' name, if deleting a process dependency"
 )
-def delete(topic, name, parent=None, child=None):
+@click.option(
+    "--cluster", help="For cluster/process relationships, the name of the cluster"
+)
+def delete(topic, name, parent=None, child=None, cluster=None):
     """
     Delete an item that is within the valid topics list and not a pre-loaded item.
     :param topic: The name of the topic.
@@ -90,20 +144,71 @@ def delete(topic, name, parent=None, child=None):
     :type name: string
     :param parent: The parent process' name, if deleting a process dependency
     :type parent: string
-    :param child: The child process' name, if deleting a process dependency
+    :param child: The child process' name, if deleting a process dependency.  For cluster/process relationships, the
+                  name of the process.
     :type child: string
+    :param cluster: For cluster/process relationships, the name of the cluster.
+    :type cluster: string
     """
     click.echo("Attempting to delete %s with name %s" % (topic, name))
-    data_store.topic_deleter(topic=topic, name=name, parent=parent, child=child)
+    data_store.topic_deleter(
+        topic=topic, name=name, parent=parent, child=child, cluster=cluster
+    )
 
 
 @main.command()
 @click.option("-t", "--topic", help="The topic being created")
 @click.option("-i", "--initial-name", help="The name that needs to be changed.")
 @click.option("-n", "--name", help="The new name for the topic.")
-def update(topic, initial_name, name):
-
+@click.option(
+    "--max-processing",
+    help="For clusters, the max processing ability allocated to the cluster",
+)
+@click.option(
+    "--max-memory", help="For clusters, the max memory allocated to the cluster"
+)
+@click.option(
+    "--processing-unit",
+    help="For clusters, the unit of processing ability (Ghz, CPU, DPU, etc.",
+)
+@click.option(
+    "--memory-unit", help="For clusters, the unit of allocated memory (MB, GB, etc."
+)
+def update(
+    topic,
+    initial_name,
+    name,
+    max_processing=None,
+    max_memory=None,
+    processing_unit=None,
+    memory_unit=None,
+):
+    """
+    Update an item that is within the valid topics list and not a pre-loaded item.
+    :param topic: The name of the topic.
+    :type topic: string
+    :param initial_name: The revised name of the topic item to be updated.
+    :type initial_name: string
+    :param name: The original name of the topic item to be updated.
+    :type name: string
+    :param max_processing: For performance clusters, the maximum processing ability allocated to the cluster
+    :type max_processing: string
+    :param max_memory: For performance clusters, the maximum memory allocated to the cluster
+    :type max_memory: string
+    :param processing_unit: For performance clusters, the unit of processing ability allocated to the cluster
+    :type processing_unit: string
+    :param memory_unit: For performance clusters, the unit of allocated memory to the cluster
+    :type memory_unit: string
+    """
     click.echo(
         "Attempting to update %s with name %s to %s" % (topic, initial_name, name)
     )
-    data_store.topic_updater(topic=topic, initial_name=initial_name, name=name)
+    data_store.topic_updater(
+        topic=topic,
+        initial_name=initial_name,
+        name=name,
+        max_processing=max_processing,
+        max_memory=max_memory,
+        processing_unit=processing_unit,
+        memory_unit=memory_unit,
+    )