Configuration File Initialization

✨ Configuration now handled thru config file - not environment variables

Working on debugging setup overwrite still.  Also realized that not all
users will be able to set up environment variables.  Switched over to a
configuration file instead.

Author: OpenDataAlex

process_tracker/cli.py

@@ -2,11 +2,10 @@
 # A set of tools to set up ProcessTracker and maintain lookup topics
 
 import click
-import os
 import logging
 
 from process_tracker.data_store import DataStore
-from process_tracker.logging import console
+from process_tracker.utilities.logging import console
 
 data_store = DataStore()
 logger = logging.getLogger('Process Tracker')
@@ -22,16 +21,16 @@ def main():
 
 
 @main.command()
-# @click.option('-o', '--overwrite', default=False, help='Wipe out the current data store and rebuild'
-#                                                        ', starting from fresh.')
-def setup():
+@click.option('-o', '--overwrite', default=False, help='Wipe out the current data store and rebuild'
+                                                       ', starting from fresh.')
+def setup(overwrite):
     """
     Initialize ProcessTracker's data store with user provided input.  If already in place, do nothing unless overwrite
     set to True.
     :return:
     """
     click.echo('Attempting to initialize data store...')
-    data_store.initialize_data_store()
+    data_store.initialize_data_store(overwrite=overwrite)
 
 
 # @main.command()

process_tracker/data_store.py

@@ -7,7 +7,8 @@
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy_utils import database_exists
 
-from process_tracker.logging import console
+from process_tracker.utilities.logging import console
+from process_tracker.utilities.settings import SettingsManager
 
 from process_tracker.models.actor import Actor
 from process_tracker.models.extract import ExtractStatus
@@ -25,12 +26,16 @@
 
 class DataStore:
 
-    def __init__(self):
+    def __init__(self, config_location=None):
         """
         Need to initialize the data store connection when starting to access the data store.
+        :param config_location: Location where Process Tracker configuration file is.
+        :type config_location: file path
         """
         self.logger = logging.getLogger(__name__)
 
+        self.config_location=config_location
+
         data_store = self.verify_and_connect_to_data_store()
         self.engine = data_store['engine']
         self.session = data_store['session']
@@ -83,8 +88,7 @@ def initialize_data_store(self, overwrite=False):
         if overwrite:
             self.logger.warn('ALERT - DATA STORE TO BE OVERWRITTEN - ALL DATA WILL BE LOST')
             self.meta.reflect()
-            self.meta.drop_all(bind=self.engine)
-            self.session.commit()
+            self.meta.drop_all()
 
             version = None
         else:
@@ -103,30 +107,27 @@ def initialize_data_store(self, overwrite=False):
             for error_type in preload_error_types:
                 self.logger.info('Adding %s' % error_type)
                 self.session.add(ErrorType(error_type_name=error_type))
-            self.session.commit()
 
             self.logger.info('Adding extract status types...')
             for extract_status_type in preload_extract_status_types:
                 self.logger.info('Adding %s' % extract_status_type)
                 self.session.add(ExtractStatus(extract_status_name=extract_status_type))
-            self.session.commit()
 
             self.logger.info('Adding process status types...')
             for process_status_type in preload_process_status_types:
                 self.logger.info('Adding %s' % process_status_type)
                 self.session.add(ProcessStatus(process_status_name=process_status_type))
-            self.session.commit()
 
             self.logger.info('Adding process types...')
             for process_type in preload_process_types:
                 self.logger.info('Adding %s' % process_type)
                 self.session.add(ProcessType(process_type_name=process_type))
-            self.session.commit()
 
             self.logger.info('Adding system keys...')
             for system_key, value in preload_system_keys:
                 self.logger.info('Adding %s' % system_key)
                 self.session.add(System(system_key=system_key, system_value=value))
+
             self.session.commit()
         else:
             self.logger.error('It appears the system has already been setup.')
@@ -322,31 +323,33 @@ def verify_and_connect_to_data_store(self):
         :return:
         """
 
-        data_store_type = os.environ.get('process_tracking_data_store_type')
-        data_store_username = os.environ.get('process_tracking_data_store_username')
-        data_store_password = os.environ.get('process_tracking_data_store_password')
-        data_store_host = os.environ.get('process_tracking_data_store_host')
-        data_store_port = os.environ.get('process_tracking_data_store_port')
-        data_store_name = os.environ.get('process_tracking_data_store_name')
+        config = SettingsManager(config_location=self.config_location).config
+
+        data_store_type = config['DEFAULT']['data_store_type']
+        data_store_username = config['DEFAULT']['data_store_username']
+        data_store_password = config['DEFAULT']['data_store_password']
+        data_store_host = config['DEFAULT']['data_store_host']
+        data_store_port = config['DEFAULT']['data_store_port']
+        data_store_name = config['DEFAULT']['data_store_name']
 
         errors = []
 
-        if data_store_type is None:
+        if data_store_type is None or data_store_type == 'None':
             errors.append(Exception('Data store type is not set.'))
 
-        if data_store_username is None:
+        if data_store_username is None or data_store_username == 'None':
             errors.append(Exception('Data store username is not set.'))
 
-        if data_store_password is None:
+        if data_store_password is None or data_store_password == 'None':
             errors.append(Exception('Data store password is not set'))
 
-        if data_store_host is None:
+        if data_store_host is None or data_store_host == 'None':
             errors.append(Exception('Data store host is not set'))
 
-        if data_store_port is None:
+        if data_store_port is None or data_store_port == 'None':
             errors.append(Exception('Data store port is not set'))
 
-        if data_store_name is None:
+        if data_store_name is None or data_store_name == 'None':
             errors.append(Exception('Data store name is not set'))
 
         if errors:
@@ -374,9 +377,9 @@ def verify_and_connect_to_data_store(self):
                 if database_exists(engine.url):
                     self.logger.info("Data store exists.  Continuing to work.")
 
-                Session = sessionmaker(bind=engine)
+                session = sessionmaker(bind=engine)
 
-                session = Session(expire_on_commit=False)
+                session = session(expire_on_commit=False)
                 session.execute("SET search_path TO %s" % data_store_name)
 
                 meta = MetaData(engine)

process_tracker/location_tracker.py

@@ -1,11 +1,10 @@
 # Location
 # For processes dealing with Extract Locations.
 import logging
-import os
 from os.path import basename, normpath
 
 from process_tracker.data_store import DataStore
-from process_tracker.logging import console
+from process_tracker.utilities.logging import console
 
 from process_tracker.models.extract import Location, LocationType
 

process_tracker/process_tracker.py

@@ -6,13 +6,12 @@
 import os
 from os.path import join
 
-import boto3
 from sqlalchemy.orm import aliased
 
 from process_tracker.data_store import DataStore
 from process_tracker.extract_tracker import ExtractTracker
 from process_tracker.location_tracker import LocationTracker
-from process_tracker.logging import console
+from process_tracker.utilities.logging import console
 
 from process_tracker.models.actor import Actor
 from process_tracker.models.extract import Extract, ExtractProcess, ExtractStatus, Location
@@ -23,20 +22,24 @@
 
 class ProcessTracker:
 
-    def __init__(self, process_name, process_type, actor_name, tool_name, sources, targets):
+    def __init__(self, process_name, process_type, actor_name, tool_name, sources, targets, config_location=None):
         """
         ProcessTracker is the primary engine for tracking data integration processes.
         :param process_name: Name of the process being tracked.
         :param actor_name: Name of the person or environment runnning the process.
         :param tool_name: Name of the tool used to run the process.
         :param sources: A single source name or list of source names for the given process.
         :type sources: list
+        :param targets: A single target name or list of target names for the given process.
+        :type targets: list
+        :param config_location: Location where Process Tracker configuration file is.
+        :type config_location: file path
         """
 
         self.logger = logging.getLogger(__name__)
         self.logger.addHandler(console)
 
-        self.data_store = DataStore()
+        self.data_store = DataStore(config_location=config_location)
         self.session = self.data_store.session
 
         self.actor = self.data_store.get_or_create_item(model=Actor, actor_name=actor_name)

process_tracker/utilities/__init__.py

@@ -0,0 +1,63 @@
+# Settings manager and configuration, both for initialization and reading.
+
+import configparser
+import os
+from pathlib import Path
+
+
+class SettingsManager:
+
+    def __init__(self, config_location=None):
+        """
+        Determine if config file exists, and if not create a basic file under the application directory.
+        :param config_location: Location where configuration file can be found.
+        :type config_location: str
+        """
+
+        self.config = configparser.ConfigParser(allow_no_value=True)
+
+        if config_location is None:
+            home = str(Path.home())
+            self.config_path = os.path.join(home, '.process_tracker/')
+            self.config_file = os.path.join(self.config_path, 'process_tracker_config.ini')
+
+        else:
+            self.config_path = os.path.join(config_location, 'process_tracker_config.ini')
+
+        exists = os.path.isfile(self.config_file)
+
+        if exists:
+            self.read_config_file()
+        else:
+            self.create_config_file()
+
+    def create_config_file(self):
+        """
+        When config file does not exist, setup a stock config file.
+        :return:
+        """
+
+        self.config['DEFAULT'] = {
+            'log_level': 'ERROR'
+        }
+
+        self.config['DEFAULT']['data_store_type'] = 'None'
+        self.config['DEFAULT']['data_store_username'] = 'None'
+        self.config['DEFAULT']['data_store_password'] = 'None'
+        self.config['DEFAULT']['data_store_host'] = 'None'
+        self.config['DEFAULT']['data_store_port'] = 'None'
+        self.config['DEFAULT']['data_store_name'] = 'None'
+
+        if not os.path.exists(self.config_path):
+            os.makedirs(self.config_path)
+
+        with open(self.config_file, 'w') as configfile:
+            self.config.write(configfile)
+
+    def read_config_file(self):
+        """
+        Read and parse the config file for use.
+        :return:
+        """
+
+        return self.config.read(self.config_file)

process_tracker/logging.py process_tracker/utilities/logging.pyprocess_tracker/logging.py renamed to process_tracker/utilities/logging.py

@@ -1,12 +1,11 @@
 import logging
-import os
 import unittest
 
 from click.testing import CliRunner
 
 from process_tracker.cli import main
 from process_tracker.data_store import DataStore
-from process_tracker.logging import console
+from process_tracker.utilities.logging import console
 
 from process_tracker.models.actor import Actor
 from process_tracker.models.extract import ExtractStatus
@@ -27,16 +26,16 @@ def setUp(self):
         self.session = self.data_store.session
         self.runner = CliRunner()
 
-    # def test_setup_overwrite(self):
-    #     """
-    #     Testing that if data store is already set up and overwrite is set to True, wipe and recreate the data store.
-    #     :return:
-    #     """
-    #     self.runner.invoke(main, 'setup -o True')
-    #
-    #     instance = self.session.query(Actor).count()
-    #
-    #     self.assertEqual(0, instance)
+    def test_setup_overwrite(self):
+        """
+        Testing that if data store is already set up and overwrite is set to True, wipe and recreate the data store.
+        :return:
+        """
+        self.runner.invoke(main, 'setup -o True')
+
+        instance = self.session.query(Actor).count()
+
+        self.assertEqual(0, instance)
 
     def test_setup_already_exists(self):
         """