Testing Setup Initialization

🐛 Data store setup still not working 100%

Moving forward with 0.1.0 beta release without the setup capability
fully working.  Worked on logging, which is now properly displaying.

Closes: #3

Author: OpenDataAlex

.gitignore

@@ -8,6 +8,8 @@
 .idea/**/tasks.xml
 .idea/**/dictionaries
 .idea/**/shelf
+log/
+
 
 # Sensitive or high-churn files
 .idea/**/dataSources/
@@ -168,3 +170,6 @@ venv.bak/
 [Ss]cripts
 pyvenv.cfg
 pip-selfcheck.json
+.idea/misc.xml
+.idea/modules.xml
+.idea/process_tracker_python.iml

process_tracker/cli.py

@@ -6,10 +6,11 @@
 import logging
 
 from process_tracker.data_store import DataStore
+from process_tracker.logging import console
 
 data_store = DataStore()
 logger = logging.getLogger('Process Tracker')
-logger.setLevel(os.environ.get('log_level', 'DEBUG'))
+logger.addHandler(console)
 
 
 @click.group()
@@ -21,11 +22,15 @@ def main():
 
 
 @main.command()
+# @click.option('-o', '--overwrite', default=False, help='Wipe out the current data store and rebuild'
+#                                                        ', starting from fresh.')
 def setup():
     """
-    Initialize ProcessTracker's data store with user provided input.  If already in place, do nothing.
+    Initialize ProcessTracker's data store with user provided input.  If already in place, do nothing unless overwrite
+    set to True.
     :return:
     """
+    click.echo('Attempting to initialize data store...')
     data_store.initialize_data_store()
 
 

process_tracker/data_store.py

@@ -4,9 +4,11 @@
 from click import ClickException
 
 from sqlalchemy import create_engine, MetaData
-from sqlalchemy.orm import sessionmaker, Session
+from sqlalchemy.orm import sessionmaker
 from sqlalchemy_utils import database_exists
 
+from process_tracker.logging import console
+
 from process_tracker.models.actor import Actor
 from process_tracker.models.extract import ExtractStatus
 from process_tracker.models.process import ErrorType, ProcessType, ProcessStatus
@@ -28,9 +30,9 @@ def __init__(self):
         Need to initialize the data store connection when starting to access the data store.
         """
         self.logger = logging.getLogger(__name__)
-        self.logger.setLevel(os.environ.get('log_level', 'DEBUG'))
 
         data_store = self.verify_and_connect_to_data_store()
+        self.engine = data_store['engine']
         self.session = data_store['session']
         self.meta = data_store['meta']
         self.data_store_type = data_store['data_store_type']
@@ -51,10 +53,10 @@ def get_or_create_item(self, model, create=True, **kwargs):
         """
 
         instance = self.session.query(model).filter_by(**kwargs).first()
+
         if instance is None:
-            self.logger.info('instance not found create = '+ str(create)+'\n')
             if create:
-                self.logger.info('creating instance\n')
+                self.logger.info('creating instance')
                 instance = model(**kwargs)
                 try:
                     self.session.add(instance)
@@ -77,13 +79,18 @@ def initialize_data_store(self, overwrite=False):
         :param overwrite: Only use if data store needs to be wiped and recreated.  Default is False.
         :type overwrite: bool
         """
-
+        self.logger.info('Attempting to initialize Process Tracker data store.')
         if overwrite:
-            self.logger.info('ALERT - DATA STORE TO BE OVERWRITTEN - ALL DATA WILL BE LOST')
+            self.logger.warn('ALERT - DATA STORE TO BE OVERWRITTEN - ALL DATA WILL BE LOST')
             self.meta.reflect()
-            self.meta.drop_all()
+            self.meta.drop_all(bind=self.engine)
+            self.session.commit()
 
-        version = self.session.query(System).filter(System.session_key == 'version').first()
+            version = None
+        else:
+
+            self.logger.debug('Obtaining system version, if exists.')
+            version = self.session.query(System).filter(System.system_key == 'version').first()
 
         if version is None:
 
@@ -94,30 +101,38 @@ def initialize_data_store(self, overwrite=False):
 
             self.logger.info('Adding error types...')
             for error_type in preload_error_types:
+                self.logger.info('Adding %s' % error_type)
                 self.session.add(ErrorType(error_type_name=error_type))
             self.session.commit()
 
             self.logger.info('Adding extract status types...')
             for extract_status_type in preload_extract_status_types:
+                self.logger.info('Adding %s' % extract_status_type)
                 self.session.add(ExtractStatus(extract_status_name=extract_status_type))
             self.session.commit()
 
             self.logger.info('Adding process status types...')
             for process_status_type in preload_process_status_types:
+                self.logger.info('Adding %s' % process_status_type)
                 self.session.add(ProcessStatus(process_status_name=process_status_type))
             self.session.commit()
 
             self.logger.info('Adding process types...')
             for process_type in preload_process_types:
+                self.logger.info('Adding %s' % process_type)
                 self.session.add(ProcessType(process_type_name=process_type))
             self.session.commit()
 
             self.logger.info('Adding system keys...')
             for system_key, value in preload_system_keys:
+                self.logger.info('Adding %s' % system_key)
                 self.session.add(System(system_key=system_key, system_value=value))
             self.session.commit()
         else:
-            self.logger.info('It appears the system has already been initialized.')
+            self.logger.error('It appears the system has already been setup.')
+            ClickException('It appears the system has already been setup.').show()
+
+        self.logger.debug('Finished the initialization check.')
 
     def topic_creator(self, topic, name):
         """
@@ -176,6 +191,7 @@ def topic_deleter(self, topic, name):
         self.logger.info('Attempting to delete %s item %s' % (topic, name))
 
         if self.topic_validator(topic=topic):
+
             if topic == 'actor':
                 item_delete = True
                 self.session.query(Actor).filter(Actor.actor_name == name).delete()
@@ -300,7 +316,6 @@ def topic_validator(self, topic):
             self.logger.error('topic type is invalid.  Please use one of the following: %s' % valid_topics.keys())
             return False
 
-
     def verify_and_connect_to_data_store(self):
         """
         Based on environment variables, create the data store connection engine.
@@ -372,6 +387,7 @@ def verify_and_connect_to_data_store(self):
                 meta = ''
 
             data_store = dict()
+            data_store['engine'] = engine
             data_store['session'] = session
             data_store['meta'] = meta
             data_store['data_store_type'] = data_store_type

process_tracker/extract_tracker.py

@@ -32,7 +32,7 @@ def __init__(self, process_run, filename, location=None, location_name=None, loc
         :type status: string
         """
         self.logger = logging.getLogger(__name__)
-        self.logger.setLevel(os.environ.get('log_level', 'ERROR'))
+        self.logger.setLevel(os.environ.get('log_level', 'DEBUG'))
 
         self.data_store = DataStore()
         self.session = self.data_store.session

process_tracker/location_tracker.py

@@ -5,14 +5,16 @@
 from os.path import basename, normpath
 
 from process_tracker.data_store import DataStore
+from process_tracker.logging import console
+
 from process_tracker.models.extract import Location, LocationType
 
 
 class LocationTracker:
 
     def __init__(self, location_path, location_name=None):
         self.logger = logging.getLogger(__name__)
-        self.logger.setLevel(os.environ.get('log_level', 'ERROR'))
+        self.logger.addHandler(console)
 
         self.data_store = DataStore()
 

process_tracker/logging.py

@@ -0,0 +1,14 @@
+import logging
+import os
+
+logging.basicConfig(level=logging.DEBUG
+                    , format="%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s")
+
+console = logging.StreamHandler()
+console.setLevel(os.environ.get('log_level', 'DEBUG'))
+
+# create formatter
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s")
+
+# set formatter
+console.setFormatter(formatter)

process_tracker/process_tracker.py

@@ -12,6 +12,7 @@
 from process_tracker.data_store import DataStore
 from process_tracker.extract_tracker import ExtractTracker
 from process_tracker.location_tracker import LocationTracker
+from process_tracker.logging import console
 
 from process_tracker.models.actor import Actor
 from process_tracker.models.extract import Extract, ExtractProcess, ExtractStatus, Location
@@ -33,7 +34,7 @@ def __init__(self, process_name, process_type, actor_name, tool_name, sources, t
         """
 
         self.logger = logging.getLogger(__name__)
-        self.logger.setLevel(os.environ.get('log_level', 'ERROR'))
+        self.logger.addHandler(console)
 
         self.data_store = DataStore()
         self.session = self.data_store.session
@@ -232,27 +233,27 @@ def register_extracts_by_location(self, location_path, location_name=None):
         """
         location = LocationTracker(location_path=location_path, location_name=location_name)
 
-        if location.location_type.location_type_name == "s3":
-            s3 = boto3.resource("s3")
-
-            path = location.location_path
-
-            if path.startswith("s3://"):
-                path = path[len("s3://")]
-
-            bucket = s3.Bucket(path)
-
-            for file in bucket.objects.all():
-                ExtractTracker(process_run=self
-                               , filename=file
-                               , location=location
-                               , status='ready')
-        else:
-            for file in os.listdir(location_path):
-                ExtractTracker(process_run=self
-                               , filename=file
-                               , location=location
-                               , status='ready')
+        # if location.location_type.location_type_name == "s3":
+        #     s3 = boto3.resource("s3")
+        #
+        #     path = location.location_path
+        #
+        #     if path.startswith("s3://"):
+        #         path = path[len("s3://")]
+        #
+        #     bucket = s3.Bucket(path)
+        #
+        #     for file in bucket.objects.all():
+        #         ExtractTracker(process_run=self
+        #                        , filename=file
+        #                        , location=location
+        #                        , status='ready')
+        # else:
+        for file in os.listdir(location_path):
+            ExtractTracker(process_run=self
+                           , filename=file
+                           , location=location
+                           , status='ready')
 
     def register_new_process_run(self):
         """

tests/test_cli.py

@@ -1,9 +1,12 @@
-from click.testing import CliRunner
+import logging
+import os
 import unittest
-import time
+
+from click.testing import CliRunner
 
 from process_tracker.cli import main
 from process_tracker.data_store import DataStore
+from process_tracker.logging import console
 
 from process_tracker.models.actor import Actor
 from process_tracker.models.extract import ExtractStatus
@@ -14,16 +17,38 @@
 
 class TestCli(unittest.TestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        cls.logger = logging.getLogger(__name__)
+        cls.logger.addHandler(console)
+
     def setUp(self):
         self.data_store = DataStore()
         self.session = self.data_store.session
         self.runner = CliRunner()
 
-    # def test_setup(self):
-    #
     # def test_setup_overwrite(self):
+    #     """
+    #     Testing that if data store is already set up and overwrite is set to True, wipe and recreate the data store.
+    #     :return:
+    #     """
+    #     self.runner.invoke(main, 'setup -o True')
     #
-    # def test_setup_already_exists(self):
+    #     instance = self.session.query(Actor).count()
+    #
+    #     self.assertEqual(0, instance)
+
+    def test_setup_already_exists(self):
+        """
+        Testing that if data store is already set up, do not attempt to initialize.
+        :return:
+        """
+
+        result = self.runner.invoke(main, ['setup'])
+
+        expected_result = 'It appears the system has already been setup.'
+
+        self.assertIn(expected_result, result.output)
 
     def test_create_actor(self):
         """
@@ -59,21 +84,16 @@ def test_create_error_type(self):
         Testing that when creating an error type record it is added.
         :return:
         """
-        print('starting the bloody test')
         result = self.runner.invoke(main, 'create -t "error type" -n "New Error Type"')
-        print(str(result.output))
-        print('got the result')
         instance = self.session.query(ErrorType).filter(ErrorType.error_type_name == 'New Error Type').first()
         try:
-            print(str(instance.error_type_name))
+            self.logger.debug(str(instance.error_type_name))
         except Exception as e:
-            print(e)
-        print('got the instance')
+            self.logger.debug(e)
         given_name = instance.error_type_name
-        print('invoking delete')
+
         self.runner.invoke(main, 'delete -t "error type" -n "New Error Type"')
-        print('delete invoked')
-        print('starting asserts')
+
         self.assertEqual('New Error Type', given_name)
 
         self.assertEqual(0, result.exit_code)
@@ -156,7 +176,7 @@ def test_delete_actor(self):
         result = self.runner.invoke(main, 'delete -t actor -n "Test Test"')
 
         instance = self.session.query(Actor).filter(Actor.actor_name == 'Test Test').first()
-        print(result.output)
+        self.logger.debug(result.output)
         self.assertEqual(None, instance)
         self.assertEqual(0, result.exit_code)