Added more robust logging

Went through and caught up logging.  Now have finer grained logging for
debugging and troubleshooting.

Also went ahead and bumped version to 0.2.0.  Will not take effect until tag is made.

Closes #22

Author: OpenDataAlex

process_tracker/__init__.py

@@ -1,4 +1,4 @@
 from process_tracker.extract_tracker import ExtractTracker
 from process_tracker.process_tracker import ProcessTracker
 
-__version__ = '0.1.0'
+__version__ = '0.2.0'

process_tracker/data_store.py

@@ -62,14 +62,21 @@ def get_or_create_item(self, model, create=True, **kwargs):
         :param kwargs: The filter criteria required to find the specific entity instance.
         :return:
         """
-
+        self.logger.debug('Attempting to obtain record.')
         instance = self.session.query(model).filter_by(**kwargs).first()
 
         if instance is None:
+
+            self.logger.debug('Record did not exist.')
+
             if create:
-                self.logger.info('creating instance')
+
+                self.logger.info('Creating instance.')
+
                 instance = model(**kwargs)
+
                 try:
+                    self.logger.debug('Committing instance.')
                     self.session.add(instance)
                 except Exception as e:
                     self.logger.error(e)
@@ -95,7 +102,7 @@ def initialize_data_store(self, overwrite=False):
             self.logger.warn('ALERT - DATA STORE TO BE OVERWRITTEN - ALL DATA WILL BE LOST')
 
             for table in Base.metadata.table_names():
-                self.logger.debug('Table will be deleted: %s' % table)
+                self.logger.info('Table will be deleted: %s' % table)
                 table.drop(self.engine)
 
             version = None
@@ -330,7 +337,7 @@ def verify_and_connect_to_data_store(self):
         Based on environment variables, create the data store connection engine.
         :return:
         """
-
+        self.logger.info('Obtaining application configuration.')
         config = SettingsManager(config_location=self.config_location).config
 
         data_store_type = config['DEFAULT']['data_store_type']
@@ -371,8 +378,13 @@ def verify_and_connect_to_data_store(self):
             engine = ''
             meta = ''
             session = ''
+            self.logger.info('Data store is supported.')
 
             if data_store_type in relational_stores:
+
+                self.logger.info('Data store is relational.')
+                self.logger.info('Data store is %s' % data_store_type)
+
                 if data_store_type == 'postgresql' \
                         or data_store_type == 'oracle'\
                         or data_store_type == 'snowflake':
@@ -398,8 +410,11 @@ def verify_and_connect_to_data_store(self):
                                                                                              , data_store_port))
 
                 if database_exists(engine.url):
+
                     self.logger.info("Data store exists.  Continuing to work.")
+
                 else:
+
                     self.logger.error('Data store does not exist.  Please create and try again.')
                     raise Exception('Data store does not exist.  Please create and try again.')
 

process_tracker/extract_tracker.py

@@ -43,19 +43,27 @@ def __init__(self, process_run, filename, location=None, location_name=None, loc
         self.filename = filename
 
         if location is not None:
+            self.logger.info('Location object provided.')
             self.location = location
         elif location_path is not None:
+            self.logger.info('Location path provided.  Creating Location object.')
             self.location = LocationTracker(location_name=location_name, location_path=location_path)
         else:
             raise Exception('A location object or location_path must be provided.')
 
+        self.logger.info('Registering extract.')
+
         self.extract = self.data_store.get_or_create_item(model=Extract
                                                           , extract_filename=filename
                                                           , extract_location_id=self.location.location.location_id)
 
         if location_path is not None:
+            self.logger.info('Location path was provided so building file path from it.')
+
             self.full_filename = join(location_path, filename)
         else:
+            self.logger.info('Location provided so building file path from it.')
+
             self.full_filename = join(self.location.location_path, self.extract.extract_filename)
 
         # Getting all status types in the event there are custom status types added later.
@@ -74,8 +82,10 @@ def __init__(self, process_run, filename, location=None, location_name=None, loc
         self.extract_process = self.retrieve_extract_process()
 
         if status is not None:
+            self.logger.info('Status was provided by user.')
             self.change_extract_status(new_status=status)
         else:
+            self.logger.info('Status was not provided.  Initializing.')
             self.extract.extract_status_id = self.extract_status_initializing
 
         self.session.commit()
@@ -85,7 +95,6 @@ def change_extract_status(self, new_status):
         Change an extract record status.
         :return:
         """
-
         status_date = datetime.now()
         if new_status in self.extract_status_types:
             self.logger.info('Setting extract status to %s' % new_status)
@@ -100,6 +109,7 @@ def change_extract_status(self, new_status):
             self.session.commit()
 
         else:
+            self.logger.error('%s is not a valid extract status type.' % new_status)
             raise Exception('%s is not a valid extract status type.  '
                             'Please add the status to extract_status_lkup' % new_status)
 
@@ -108,7 +118,9 @@ def get_extract_status_types(self):
         Get list of process status types and return dictionary.
         :return:
         """
-        status_types = {}
+        self.logger.info('Obtaining extract status types.')
+
+        status_types = dict()
 
         for record in self.session.query(ExtractStatus):
             status_types[record.extract_status_name] = record.extract_status_id
@@ -121,14 +133,16 @@ def retrieve_extract_process(self):
         :return:
         """
 
+        self.logger.info('Associating extract to given process.')
+
         extract_process = self.data_store.get_or_create_item(model=ExtractProcess
                                                              , extract_tracking_id=self.extract.extract_id
                                                              , process_tracking_id=self.process_run.process_tracking_run
                                                              .process_tracking_id)
 
         # Only need to set to 'initializing' when it's the first time a process run is trying to work with files.
         if extract_process.extract_process_status_id is None:
-
+            self.logger.info('Extract process status must also be set.  Initializing.')
             extract_process.extract_process_status_id = self.extract_status_initializing
             self.session.commit()
 

process_tracker/location_tracker.py

@@ -24,11 +24,16 @@ def __init__(self, location_path, location_name=None):
         self.location_path = location_path.lower()
 
         if location_name is None:
+            self.logger.info('Location name not provided.  Generating.')
             self.location_name = self.derive_location_name()
         else:
+            self.logger.info('Using provided location name: %s' % location_name)
             self.location_name = location_name
 
         self.location_type = self.derive_location_type()
+
+        self.logger.info('Registering extract location.')
+
         self.location = self.data_store.get_or_create_item(model=Location
                                                            , location_name=self.location_name
                                                            , location_path=location_path
@@ -48,10 +53,11 @@ def derive_location_name(self):
 
         if "s3" in self.location_path:
             # If the path is an S3 Bucket, prefix to name.
-
+            self.logger.info('Location appears to be s3 related.  Setting prefix.')
             location_prefix = "s3"
 
         if location_prefix is not None:
+            self.logger.info('Location prefix provided.  Appending to location name.')
             location_name = location_prefix + " - "
 
         location_name += basename(normpath(self.location_path))
@@ -66,10 +72,15 @@ def derive_location_type(self):
 
         if "s3" in self.location_path or "s3" in self.location_name:
 
+            self.logger.info('Location appears to be s3 related.  Setting type to s3.')
+
             location_type = self.data_store.get_or_create_item(model=LocationType
                                                                , location_type_name="s3")
 
         else:
+
+            self.logger.info('Location did not match special types.  Assuming local directory path.')
+
             location_type = self.data_store.get_or_create_item(model=LocationType
                                                                , location_type_name="local directory")