Reconfiguring code to prepare for packaging

👷  Travis-CI and Coveralls working
🏗️ Moved data_store initialization to DataStore

Verifying that build stack is working, sans package deploy to PyPi.  Initial configuration for PyPi is in travis config, but disabled.
This commit is the first steps for getting the package ready for deploy.
Major refactoring had to be done to ensure that code is only run when
called as opposed to on import.

Closes: #5

Author: OpenDataAlex

.travis.yml

@@ -1,17 +1,27 @@
 dist: xenial
 language: python
 python:
-  - "3.7"
+- '3.7'
 install:
-  - "make"
+- make
 services:
-  - postgresql
+- postgresql
 script:
-  - make test
+- make test
 after_success:
-  - coveralls
+- coveralls
 before_script:
-  - psql -f dbscripts/postgresql_account_create.sql -U postgres
-  - psql -f dbscripts/postgresql_process_tracker.sql process_tracking -U postgres
+- psql -f dbscripts/postgresql_account_create.sql -U postgres
+- psql -f dbscripts/postgresql_process_tracker.sql process_tracking -U postgres
 env:
-  - process_tracking_data_store_host=localhost process_tracking_data_store_port=5432 process_tracking_data_store_name=process_tracking process_tracking_data_store_type=postgresql process_tracking_data_store_password=Testing1! process_tracking_data_store_username=pt_admin
+- process_tracking_data_store_host=localhost process_tracking_data_store_port=5432
+  process_tracking_data_store_name=process_tracking process_tracking_data_store_type=postgresql
+  process_tracking_data_store_password=Testing1! process_tracking_data_store_username=pt_admin
+#deploy:
+#  matrix:
+#  - provider: pypi
+#  - user: opendataalex
+#  - password:
+#      secure: jYe8ygGYuJox5f1vQvraIDNlR2Yk/+/afIXdsxaxRi/QN135rIHxRih24iYv8ZQewfW+w4CveEPz2pr0ew5NG8hm2MD5hQZI/A2QE4BC53Tr3xMAWSdPQgcI1rX4Ieu6BHTS3jet212yLuBk41GdASKztz+s/CVvVqT0A29U1b85OVzoVHz6OTgaM1m0pKeJE6Ho66r5O6J7ClZfEY4RJ8jROmrT7ufW3hEs/VQXwfAUWcjy6l5H5VYTLa/g94MgQ+0WmgcdUbB8s3c2q7n4RiaVEowZOLtBmRElDIPiLJC8JGAIFKcf5W8pzXK/ZhZhY3ppgmBvRVeyPdiAwjysionzl4vIlNxC28acI3HzvJbAdvwPsuMD+pnTo/LKRqqvGey2NSnMPmzGXxzqVUqQ5Wa1Yzdcqg8mbWHFRw9icMWKM84CnDBvqkmJcfnj5R4X6Rc77mr9jc5LMMBBblNtY7dQc2AooframujGA2mP1xABQ9vOSg89MdnlhHrgEK6m+Pz0S2hhlsGRw9yMF9L4Q1ie7xKeE49L3HKZSjKEgSMv9y9RbqnmW32F0UoKgWkCJE8DV9wbZe5Htn+lpwmXI0vu+cxnqsiZvG8wjm+klJGn5F84OFTN3zQOC/kIBXQXvutZYtgF0vofdCmKQzK9istl16Eur1r4NCbov3oM97o=
+#    on:
+#      tags: true

README.md

@@ -3,6 +3,7 @@ ProcessTracker - Python
 Data integration process management made easy!
 
 [![Coverage Status](https://coveralls.io/repos/github/OpenDataAlex/process_tracker_python/badge.svg?branch=master)](https://coveralls.io/github/OpenDataAlex/process_tracker_python?branch=master)
+[![Build Status](https://travis-ci.org/OpenDataAlex/process_tracker_python.svg?branch=master)](https://travis-ci.org/OpenDataAlex/process_tracker_python)
 
 This is the Python implementation of the ProcessTracker framework.  ProcessTracker builds a standard framework that is
 tool agnostic.  If you are working with data integration/cleansing processes within Python (i.e. using PySpark, Pandas, etc.)

process_tracker/__init__.py

@@ -1,78 +0,0 @@
-import os
-import logging
-
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-log = logging.getLogger(__name__)
-
-"""
-Based on environment variables, create the data store connection engine.
-:return:
-"""
-
-data_store_type = os.environ.get('process_tracking_data_store_type')
-data_store_username = os.environ.get('process_tracking_data_store_username')
-data_store_password = os.environ.get('process_tracking_data_store_password')
-data_store_host = os.environ.get('process_tracking_data_store_host')
-data_store_port = os.environ.get('process_tracking_data_store_port')
-data_store_name = os.environ.get('process_tracking_data_store_name')
-
-data_store_error_flag = False
-
-if data_store_type is None:
-    raise Exception('Data store type is not set.')
-    data_store_error_flag = True
-
-if data_store_username is None:
-    raise Exception('Data store username is not set.')
-    data_store_error_flag = True
-
-if data_store_password is None:
-    raise Exception('Data store password is not set')
-    data_store_error_flag = True
-
-if data_store_host is None:
-    raise Exception('Data store host is not set')
-    data_store_error_flag = True
-
-if data_store_port is None:
-    raise Exception('Data store port is not set')
-    data_store_error_flag = True
-
-if data_store_name is None:
-    raise Exception('Data store name is not set')
-    data_store_error_flag = True
-
-if data_store_error_flag:
-    raise Exception('Data store has not been properly configured.  Please read how to set up the Process '
-                    'Tracking data store by going to: <insert read the docs url here>')
-
-relational_stores = ['postgresql']
-nonrelational_stores = []
-
-supported_data_stores = relational_stores + nonrelational_stores
-
-if data_store_type in supported_data_stores:
-
-    if data_store_type in relational_stores:
-        if data_store_type == 'postgresql':
-            engine = create_engine(data_store_type + '://' + data_store_username + ':' + data_store_password
-                                   + '@' + data_store_host + '/' + data_store_name)
-
-            log.info("Attempting to connect to datastore %s, found at %s:%s" % (data_store_name
-                                                                                , data_store_host
-                                                                                , data_store_port))
-
-        Session = sessionmaker(bind=engine)
-
-        session = Session(expire_on_commit=False)
-        session.execute("SET search_path TO %s" % data_store_name)
-
-    elif data_store_type in nonrelational_stores:
-        Session = ''
-
-
-else:
-    raise Exception('Invalid data store type provided.  Please use: ' + ", ".join(supported_data_stores))
-    exit()

process_tracker/data_store.py

@@ -1,10 +1,26 @@
-from process_tracker import session
+import os
+import logging
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
 
 
 class DataStore:
 
-    @staticmethod
-    def get_or_create(model, create=True, **kwargs):
+    def __init__(self):
+        """
+        Need to initialize the data store connection when starting to access the data store.
+        """
+        self.logger = logging.getLogger(__name__)
+
+        data_store = self.verify_and_connect_to_data_store()
+        self.session = data_store['session']
+        self.data_store_type = data_store['data_store_type']
+        self.data_store_host = data_store['data_store_host']
+        self.data_store_port = data_store['data_store_port']
+        self.data_store_name = data_store['data_store_name']
+
+    def get_or_create(self, model, create=True, **kwargs):
         """
         Testing if an entity instance exists or not.  If does, return entity key.  If not, create entity instance
         and return key.
@@ -16,16 +32,97 @@ def get_or_create(model, create=True, **kwargs):
         :return:
         """
 
-        instance = session.query(model).filter_by(**kwargs).first()
+        instance = self.session.query(model).filter_by(**kwargs).first()
 
         if not instance:
 
             if create:
                 instance = model(**kwargs)
-                session.add(instance)
-                session.commit()
+                self.session.add(instance)
+                self.session.commit()
 
             else:
                 raise Exception('There is no record match in %s .' % model.__tablename__)
 
         return instance
+
+    def verify_and_connect_to_data_store(self):
+        """
+        Based on environment variables, create the data store connection engine.
+        :return:
+        """
+
+        data_store_type = os.environ.get('process_tracking_data_store_type')
+        data_store_username = os.environ.get('process_tracking_data_store_username')
+        data_store_password = os.environ.get('process_tracking_data_store_password')
+        data_store_host = os.environ.get('process_tracking_data_store_host')
+        data_store_port = os.environ.get('process_tracking_data_store_port')
+        data_store_name = os.environ.get('process_tracking_data_store_name')
+
+        data_store_error_flag = False
+
+        if data_store_type is None:
+            raise Exception('Data store type is not set.')
+            data_store_error_flag = True
+
+        if data_store_username is None:
+            raise Exception('Data store username is not set.')
+            data_store_error_flag = True
+
+        if data_store_password is None:
+            raise Exception('Data store password is not set')
+            data_store_error_flag = True
+
+        if data_store_host is None:
+            raise Exception('Data store host is not set')
+            data_store_error_flag = True
+
+        if data_store_port is None:
+            raise Exception('Data store port is not set')
+            data_store_error_flag = True
+
+        if data_store_name is None:
+            raise Exception('Data store name is not set')
+            data_store_error_flag = True
+
+        if data_store_error_flag:
+            raise Exception('Data store has not been properly configured.  Please read how to set up the Process '
+                            'Tracking data store by going to: <insert read the docs url here>')
+
+        relational_stores = ['postgresql']
+        nonrelational_stores = []
+
+        supported_data_stores = relational_stores + nonrelational_stores
+
+        if data_store_type in supported_data_stores:
+
+            if data_store_type in relational_stores:
+                if data_store_type == 'postgresql':
+                    engine = create_engine(data_store_type + '://' + data_store_username + ':' + data_store_password
+                                           + '@' + data_store_host + '/' + data_store_name)
+
+                    self.logger.info("Attempting to connect to datastore %s, found at %s:%s" % (data_store_name
+                                                                                        , data_store_host
+                                                                                        , data_store_port))
+
+                Session = sessionmaker(bind=engine)
+
+                session = Session(expire_on_commit=False)
+                session.execute("SET search_path TO %s" % data_store_name)
+
+            elif data_store_type in nonrelational_stores:
+                Session = ''
+
+            data_store = dict()
+            data_store['session'] = session
+            data_store['data_store_type'] = data_store_type
+            data_store['data_store_host'] = data_store_host
+            data_store['data_store_port'] = data_store_port
+            data_store['data_store_name'] = data_store_name
+            data_store['data_store_username'] = data_store_username
+            data_store['data_store_password'] = data_store_password
+
+            return data_store
+        else:
+            raise Exception('Invalid data store type provided.  Please use: ' + ", ".join(supported_data_stores))
+            exit()

process_tracker/extract_tracker.py

@@ -3,10 +3,9 @@
 from datetime import datetime
 from os.path import basename, normpath
 
-from process_tracker import session
 from process_tracker.data_store import DataStore
 
-from models.extract import Extract, ExtractProcess, ExtractStatus, Location
+from process_tracker.models.extract import Extract, ExtractProcess, ExtractStatus, Location
 
 
 class ExtractTracker:
@@ -26,6 +25,7 @@ def __init__(self, process_run, filename, location_path, location_name=None):
         :type location_name: string
         """
         self.data_store = DataStore()
+        self.session = self.data_store.session
         self.process_run = process_run
 
         if location_name is None:
@@ -59,7 +59,7 @@ def __init__(self, process_run, filename, location_path, location_name=None):
         self.extract_process = self.retrieve_extract_process()
 
         self.extract.extract_status_id = self.extract_status_initializing
-        session.commit()
+        self.session.commit()
 
     def change_extract_status(self, new_status):
         """
@@ -75,7 +75,7 @@ def change_extract_status(self, new_status):
             self.extract_process.extract_process_status_id = new_status
             self.extract_process.extract_process_event_date_time = status_date
 
-            session.commit()
+            self.session.commit()
 
         else:
             raise Exception('%s is not a valid extract status type.  '
@@ -109,15 +109,14 @@ def derive_location_name(location_path):
 
         return location_name
 
-    @staticmethod
-    def get_extract_status_types():
+    def get_extract_status_types(self):
         """
         Get list of process status types and return dictionary.
         :return:
         """
         status_types = {}
 
-        for record in session.query(ExtractStatus):
+        for record in self.session.query(ExtractStatus):
             status_types[record.extract_status_name] = record.extract_status_id
 
         return status_types
@@ -137,6 +136,6 @@ def retrieve_extract_process(self):
         if extract_process.extract_process_status_id is None:
 
             extract_process.extract_process_status_id = self.extract_status_initializing
-            session.commit()
+            self.session.commit()
 
         return extract_process

models/__init__.py process_tracker/models/__init__.pymodels/__init__.py renamed to process_tracker/models/__init__.py

@@ -2,8 +2,7 @@
 # Models for Actor entities
 
 from sqlalchemy import Column, Integer, Sequence, String
-from sqlalchemy_utils.types import uuid
-from models.model_base import Base
+from process_tracker.models.model_base import Base
 
 
 class Actor(Base):

models/actor.py process_tracker/models/actor.pymodels/actor.py renamed to process_tracker/models/actor.py

@@ -6,7 +6,7 @@
 from sqlalchemy import Column, DateTime, ForeignKey, Integer, Sequence, String
 from sqlalchemy.orm import relationship
 
-from models.model_base import Base
+from process_tracker.models.model_base import Base
 
 
 class ExtractStatus(Base):

models/extract.py process_tracker/models/extract.pymodels/extract.py renamed to process_tracker/models/extract.py

@@ -5,8 +5,7 @@
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, Sequence, String
 from sqlalchemy.orm import relationship
 
-from models.model_base import default_date, Base
-from models.extract import Extract
+from process_tracker.models.model_base import default_date, Base
 
 
 class ErrorType(Base):