process_tracker_python-30 Extract Lookups should return Extract objects

All the extract file finders now return extract objects so they can be
modified as they are used.  A helper method was also added that provides
the full filename of a given extract (called full_filepath).

Also realized that I missed adding string representations of a bunch of
the data models.

Closes #30

Author: OpenDataAlex

process_tracker/__init__.py

@@ -1,4 +1,4 @@
 from process_tracker.extract_tracker import ExtractTracker
 from process_tracker.process_tracker import ProcessTracker
 
-__version__ = '0.1.6'
+__version__ = '0.2.0'

process_tracker/data_store.py

@@ -62,14 +62,21 @@ def get_or_create_item(self, model, create=True, **kwargs):
         :param kwargs: The filter criteria required to find the specific entity instance.
         :return:
         """
-
+        self.logger.debug('Attempting to obtain record.')
         instance = self.session.query(model).filter_by(**kwargs).first()
 
         if instance is None:
+
+            self.logger.debug('Record did not exist.')
+
             if create:
-                self.logger.info('creating instance')
+
+                self.logger.info('Creating instance.')
+
                 instance = model(**kwargs)
+
                 try:
+                    self.logger.debug('Committing instance.')
                     self.session.add(instance)
                 except Exception as e:
                     self.logger.error(e)
@@ -95,7 +102,7 @@ def initialize_data_store(self, overwrite=False):
             self.logger.warn('ALERT - DATA STORE TO BE OVERWRITTEN - ALL DATA WILL BE LOST')
 
             for table in Base.metadata.table_names():
-                self.logger.debug('Table will be deleted: %s' % table)
+                self.logger.info('Table will be deleted: %s' % table)
                 table.drop(self.engine)
 
             version = None
@@ -330,7 +337,7 @@ def verify_and_connect_to_data_store(self):
         Based on environment variables, create the data store connection engine.
         :return:
         """
-
+        self.logger.info('Obtaining application configuration.')
         config = SettingsManager(config_location=self.config_location).config
 
         data_store_type = config['DEFAULT']['data_store_type']
@@ -371,8 +378,13 @@ def verify_and_connect_to_data_store(self):
             engine = ''
             meta = ''
             session = ''
+            self.logger.info('Data store is supported.')
 
             if data_store_type in relational_stores:
+
+                self.logger.info('Data store is relational.')
+                self.logger.info('Data store is %s' % data_store_type)
+
                 if data_store_type == 'postgresql' \
                         or data_store_type == 'oracle'\
                         or data_store_type == 'snowflake':
@@ -398,8 +410,11 @@ def verify_and_connect_to_data_store(self):
                                                                                              , data_store_port))
 
                 if database_exists(engine.url):
+
                     self.logger.info("Data store exists.  Continuing to work.")
+
                 else:
+
                     self.logger.error('Data store does not exist.  Please create and try again.')
                     raise Exception('Data store does not exist.  Please create and try again.')
 

process_tracker/extract_tracker.py

@@ -43,19 +43,27 @@ def __init__(self, process_run, filename, location=None, location_name=None, loc
         self.filename = filename
 
         if location is not None:
+            self.logger.info('Location object provided.')
             self.location = location
         elif location_path is not None:
+            self.logger.info('Location path provided.  Creating Location object.')
             self.location = LocationTracker(location_name=location_name, location_path=location_path)
         else:
             raise Exception('A location object or location_path must be provided.')
 
+        self.logger.info('Registering extract.')
+
         self.extract = self.data_store.get_or_create_item(model=Extract
                                                           , extract_filename=filename
                                                           , extract_location_id=self.location.location.location_id)
 
         if location_path is not None:
+            self.logger.info('Location path was provided so building file path from it.')
+
             self.full_filename = join(location_path, filename)
         else:
+            self.logger.info('Location provided so building file path from it.')
+
             self.full_filename = join(self.location.location_path, self.extract.extract_filename)
 
         # Getting all status types in the event there are custom status types added later.
@@ -74,8 +82,10 @@ def __init__(self, process_run, filename, location=None, location_name=None, loc
         self.extract_process = self.retrieve_extract_process()
 
         if status is not None:
+            self.logger.info('Status was provided by user.')
             self.change_extract_status(new_status=status)
         else:
+            self.logger.info('Status was not provided.  Initializing.')
             self.extract.extract_status_id = self.extract_status_initializing
 
         self.session.commit()
@@ -85,7 +95,6 @@ def change_extract_status(self, new_status):
         Change an extract record status.
         :return:
         """
-
         status_date = datetime.now()
         if new_status in self.extract_status_types:
             self.logger.info('Setting extract status to %s' % new_status)
@@ -100,6 +109,7 @@ def change_extract_status(self, new_status):
             self.session.commit()
 
         else:
+            self.logger.error('%s is not a valid extract status type.' % new_status)
             raise Exception('%s is not a valid extract status type.  '
                             'Please add the status to extract_status_lkup' % new_status)
 
@@ -108,7 +118,9 @@ def get_extract_status_types(self):
         Get list of process status types and return dictionary.
         :return:
         """
-        status_types = {}
+        self.logger.info('Obtaining extract status types.')
+
+        status_types = dict()
 
         for record in self.session.query(ExtractStatus):
             status_types[record.extract_status_name] = record.extract_status_id
@@ -121,14 +133,16 @@ def retrieve_extract_process(self):
         :return:
         """
 
+        self.logger.info('Associating extract to given process.')
+
         extract_process = self.data_store.get_or_create_item(model=ExtractProcess
                                                              , extract_tracking_id=self.extract.extract_id
                                                              , process_tracking_id=self.process_run.process_tracking_run
                                                              .process_tracking_id)
 
         # Only need to set to 'initializing' when it's the first time a process run is trying to work with files.
         if extract_process.extract_process_status_id is None:
-
+            self.logger.info('Extract process status must also be set.  Initializing.')
             extract_process.extract_process_status_id = self.extract_status_initializing
             self.session.commit()
 

process_tracker/location_tracker.py

@@ -24,11 +24,16 @@ def __init__(self, location_path, location_name=None):
         self.location_path = location_path.lower()
 
         if location_name is None:
+            self.logger.info('Location name not provided.  Generating.')
             self.location_name = self.derive_location_name()
         else:
+            self.logger.info('Using provided location name: %s' % location_name)
             self.location_name = location_name
 
         self.location_type = self.derive_location_type()
+
+        self.logger.info('Registering extract location.')
+
         self.location = self.data_store.get_or_create_item(model=Location
                                                            , location_name=self.location_name
                                                            , location_path=location_path
@@ -48,10 +53,11 @@ def derive_location_name(self):
 
         if "s3" in self.location_path:
             # If the path is an S3 Bucket, prefix to name.
-
+            self.logger.info('Location appears to be s3 related.  Setting prefix.')
             location_prefix = "s3"
 
         if location_prefix is not None:
+            self.logger.info('Location prefix provided.  Appending to location name.')
             location_name = location_prefix + " - "
 
         location_name += basename(normpath(self.location_path))
@@ -66,10 +72,15 @@ def derive_location_type(self):
 
         if "s3" in self.location_path or "s3" in self.location_name:
 
+            self.logger.info('Location appears to be s3 related.  Setting type to s3.')
+
             location_type = self.data_store.get_or_create_item(model=LocationType
                                                                , location_type_name="s3")
 
         else:
+
+            self.logger.info('Location did not match special types.  Assuming local directory path.')
+
             location_type = self.data_store.get_or_create_item(model=LocationType
                                                                , location_type_name="local directory")
 

tests/test_process_tracker.py

@@ -432,7 +432,7 @@ def test_register_new_process_run_exception(self):
             # Running registration a second time to mimic job being run twice
             self.process_tracker.register_new_process_run()
 
-        return self.assertTrue('The process Testing Process Tracking Initialization '
+        return self.assertTrue('Process Testing Process Tracking Initialization '
                                'is currently running.' in str(context.exception))
 
     def test_register_new_process_run_with_previous_run(self):