process_tracker_python-144 Add Process Dependency Classifier

✨ Dependency type added

Extract and Process dependencies can now be classified by type.

Author: Alex Meadows

dbscripts/mysql_process_tracker.sql

@@ -266,7 +266,7 @@ create table process_tracker.extract_dependency
 (
 	parent_extract_id int not null,
 	child_extract_id int not null,
-	dependency_type_id int default 0 not null,
+	dependency_type_id int default 1 not null,
 	created_date_time timestamp default CURRENT_TIMESTAMP not null,
 	created_by int default 0 not null,
 	update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null,
@@ -389,7 +389,7 @@ create table process_dependency
 (
 	parent_process_id int not null,
 	child_process_id int not null,
-	dependency_type_id int default 0 not null,
+	dependency_type_id int default 1 not null,
 	created_date_time timestamp default CURRENT_TIMESTAMP not null,
 	created_by int default 0 not null,
 	update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null,

dbscripts/mysql_process_tracker_defaults.sql

@@ -55,7 +55,7 @@ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name)
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal');
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
 
-INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (0, 'Undefined');
-INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Hard');
-INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (4, 'Soft');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft');
 

dbscripts/postgresql_process_tracker.sql

@@ -421,7 +421,7 @@ create table process_dependency
 			references process,
 	constraint process_dependency_pk
 		primary key (child_process_id, parent_process_id),
-	dependency_type_id int default 0 not null
+	dependency_type_id int default 1 not null
 	    constraint process_dependency_fk03
 		    references dependency_type_lkup,
 	created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null,
@@ -1054,7 +1054,7 @@ create table extract_dependency
 	child_extract_id integer not null
 		constraint extract_dependency_fk02
 			references extract_tracking,
-	dependency_type_id int default 0 not null
+	dependency_type_id int default 1 not null
 	    constraint extract_dependency_fk03
 		    references dependency_type_lkup,
 	constraint extract_dependency_pk

dbscripts/postgresql_process_tracker_defaults.sql

@@ -54,6 +54,6 @@ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name)
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal');
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
 
-INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (0, 'Undefined');
-INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Hard');
-INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Soft');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft');

process_tracker/utilities/aws_utilities.py

@@ -15,6 +15,11 @@ def __init__(self):
         self.logger = logging.getLogger(__name__)
         self.logger.setLevel(self.log_level)
 
+        logging.getLogger("boto3").setLevel(logging.CRITICAL)
+        logging.getLogger("botocore").setLevel(logging.CRITICAL)
+        logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
+        logging.getLogger("urllib3").setLevel(logging.CRITICAL)
+
         self.s3 = boto3.resource("s3")
 
         self.url_match = re.compile(