Merge pull request #121 from OpenDataAlex/process_tracker_python-118

Process tracker python 118

Author: OpenDataAlex

dbscripts/mysql_process_tracker.sql

@@ -489,7 +489,7 @@ create table process_contact
 		foreign key (contact_id) references contact_lkup (contact_id)
 );
 
-create table source_object_attribute
+create table source_object_attribute_lkup
 (
 	source_object_attribute_id int auto_increment
 		primary key,
@@ -504,11 +504,11 @@ create table source_object_attribute
 	default_value_number decimal null,
 	is_key boolean default FALSE not null,
 	is_filter boolean default FALSE not null,
-	constraint source_object_attribute_udx01
+	constraint source_object_attribute_lkup_udx01
 		unique (source_object_id, source_object_attribute_name),
-	constraint source_object_attribute_fk01
+	constraint source_object_attribute_lkup_fk01
 		foreign key (source_object_id) references source_object_lkup (source_object_id),
-	constraint source_object_attribute_fk02
+	constraint source_object_attribute_lkup_fk02
 		foreign key (data_type_id) references data_type_lkup (data_type_id)
 );
 
@@ -522,7 +522,7 @@ create table process_source_object_attribute
 	constraint process_source_object_attribute_fk01
 		foreign key (process_id) references process (process_id),
 	constraint process_source_object_attribute_fk02
-		foreign key (source_object_attribute_id) references source_object_attribute (source_object_attribute_id)
+		foreign key (source_object_attribute_id) references source_object_attribute_lkup (source_object_attribute_id)
 );
 
 create table if not exists process_target_object_attribute
@@ -535,7 +535,7 @@ create table if not exists process_target_object_attribute
 	constraint process_target_object_attribute_fk01
 		foreign key (process_id) references process (process_id),
 	constraint process_target_object_attribute_fk02
-		foreign key (target_object_attribute_id) references source_object_attribute (source_object_attribute_id)
+		foreign key (target_object_attribute_id) references source_object_attribute_lkup (source_object_attribute_id)
 );
 
 create table filter_type_lkup
@@ -564,7 +564,7 @@ create table process_filter
 	constraint process_filter_fk01
 		foreign key (process_id) references process (process_id),
 	constraint process_filter_fk02
-		foreign key (source_object_attribute_id) references source_object_attribute (source_object_attribute_id),
+		foreign key (source_object_attribute_id) references source_object_attribute_lkup (source_object_attribute_id),
 	constraint process_filter_fk03
 		foreign key (filter_type_id) references filter_type_lkup (filter_type_id)
 );

dbscripts/postgresql_process_tracker.sql

@@ -727,18 +727,18 @@ create table process_contact
 
 alter table process_contact owner to pt_admin;
 
-create table process_tracker.source_object_attribute
+create table process_tracker.source_object_attribute_lkup
 (
 	source_object_attribute_id serial not null
-		constraint source_object_attribute_pk
+		constraint source_object_attribute_lkup_pk
 			primary key,
 	source_object_attribute_name varchar(250) not null,
 	source_object_id integer
-		constraint source_object_attribute_fk01
+		constraint source_object_attribute_lkup_fk01
 			references process_tracker.source_object_lkup,
 	attribute_path varchar(750),
 	data_type_id integer
-		constraint source_object_attribute_fk02
+		constraint source_object_attribute_lkup_fk02
 			references process_tracker.data_type_lkup,
 	data_length integer,
 	data_decimal integer,
@@ -749,10 +749,10 @@ create table process_tracker.source_object_attribute
 	is_filter boolean default false not null
 );
 
-alter table process_tracker.source_object_attribute owner to pt_admin;
+alter table process_tracker.source_object_attribute_lkup owner to pt_admin;
 
-create unique index source_object_attribute_udx01
-	on process_tracker.source_object_attribute (source_object_id, source_object_attribute_name);
+create unique index source_object_attribute_lkup_udx01
+	on process_tracker.source_object_attribute_lkup (source_object_id, source_object_attribute_name);
 
 create table process_tracker.process_target_object_attribute
 (
@@ -761,7 +761,7 @@ create table process_tracker.process_target_object_attribute
 			references process_tracker.process,
 	target_object_attribute_id integer not null
 		constraint process_target_object_attribute_fk02
-			references process_tracker.source_object_attribute,
+			references process_tracker.source_object_attribute_lkup,
 	target_object_attribute_alias varchar(250),
 	target_object_attribute_expression varchar(250),
 	constraint process_target_object_attribute_pk
@@ -777,7 +777,7 @@ create table process_tracker.process_source_object_attribute
 			references process_tracker.process,
 	source_object_attribute_id integer not null
 		constraint process_source_object_attribute_fk02
-			references process_tracker.source_object_attribute,
+			references process_tracker.source_object_attribute_lkup,
 	source_object_attribute_alias varchar(250),
 	source_object_attribute_expression varchar(250),
 	constraint process_source_object_attribute_pk
@@ -797,7 +797,7 @@ create table process_tracker.process_filter
 			references process_tracker.process,
 	source_object_attribute_id integer not null
 		constraint process_filter_fk02
-			references process_tracker.source_object_attribute,
+			references process_tracker.source_object_attribute_lkup,
 	filter_type_id integer not null
 		constraint process_filter_fk03
 			references process_tracker.filter_type_lkup,

process_tracker/models/process.py

@@ -250,7 +250,7 @@ class ProcessFilter(Base):
     source_object_attribute_id = Column(
         Integer,
         ForeignKey(
-            "process_tracker.source_object_attribute.source_object_attribute_id"
+            "process_tracker.source_object_attribute_lkup.source_object_attribute_id"
         ),
         nullable=False,
     )
@@ -344,7 +344,7 @@ class ProcessSourceObjectAttribute(Base):
     source_object_attribute_id = Column(
         Integer,
         ForeignKey(
-            "process_tracker.source_object_attribute.source_object_attribute_id"
+            "process_tracker.source_object_attribute_lkup.source_object_attribute_id"
         ),
         primary_key=True,
         nullable=False,
@@ -430,7 +430,7 @@ class ProcessTargetObjectAttribute(Base):
     target_object_attribute_id = Column(
         Integer,
         ForeignKey(
-            "process_tracker.source_object_attribute.source_object_attribute_id"
+            "process_tracker.source_object_attribute_lkup.source_object_attribute_id"
         ),
         primary_key=True,
         nullable=False,

process_tracker/models/source.py

@@ -196,7 +196,7 @@ def __repr__(self):
 
 class SourceObjectAttribute(Base):
 
-    __tablename__ = "source_object_attribute"
+    __tablename__ = "source_object_attribute_lkup"
     __table_args__ = {"schema": "process_tracker"}
 
     source_object_attribute_id = Column(

process_tracker/process_tracker.py

@@ -810,7 +810,7 @@ def register_new_process_run(self):
                 last_run_id=last_run.process_run_id,
             ):
                 self.logger.error(
-                    "Process is on hold due to number of concurrent failures or previous run is in on hold status."
+                    "Process is on hold due to number of concurrent failures or previous run is in on-hold status."
                 )
                 new_run_flag = False
 
@@ -833,7 +833,8 @@ def register_new_process_run(self):
 
         else:
             raise Exception(
-                "The process %s is currently running or on hold." % self.process_name
+                "The process %s is currently %s."
+                % (self.process_name, last_run.status.process_status_name)
             )
 
     def register_process_dataset_types(self, dataset_types):

tests/test_process_tracker.py

@@ -750,7 +750,7 @@ def test_process_on_hold_max_failures(self):
             )
 
         self.assertTrue(
-            "The process On Hold Max Failures Test is currently running or on_hold."
+            "The process On Hold Max Failures Test is currently on hold."
             in str(context.exception)
         )
 
@@ -809,7 +809,7 @@ def test_process_on_hold_previous_run_on_hold(self):
             )
 
         self.assertTrue(
-            "The process On Hold Previous Run Test is currently running or on hold."
+            "The process On Hold Previous Run Test is currently on hold."
             in str(context.exception)
         )
 
@@ -903,6 +903,7 @@ def test_register_extracts_by_location_local(self):
         "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
         "Skipping this test on Travis CI.",
     )
+    @unittest.skip("Issue with hanging queries on database.")
     @mock_s3
     def test_register_extracts_by_location_s3(self):
         """
@@ -1034,7 +1035,7 @@ def test_register_new_process_run_exception(self):
             self.process_tracker.register_new_process_run()
 
         return self.assertTrue(
-            "The process Testing Process Tracking Initialization is currently running or on hold."
+            "The process Testing Process Tracking Initialization is currently running."
             in str(context.exception)
         )