process_tracker_python-1 Add support for other relational databases

✨ Added Support for Oracle and SQL Server

In addition to Postgresql, MySQL, and Snowflake, have now added support
for Oracle and SQL Server.  The Oracle, SQL Server, and Snowflake are
not tested at this time - working on setting up local data stores.
At the moment, can't test them with Travis as they are not available.

I do plan on testing them thoroughly locally like I have with Postgres
and MySQL.  One thing I've noticed is that likely due to the processing
differences between systems, tests that pass locally may fail so there is
some (very slight) risk something may go bork without more higher
powered testing.

Closes #1

Author: OpenDataAlex

Pipfile

@@ -19,6 +19,8 @@ psycopg2-binary="*"
 google-compute-engine = "*"
 snowflake-sqlalchemy = "*"
 pymysql = "*"
+cx-oracle = "*"
+pymssql = "*"
 
 [requires]
 python_version = "3.7"

Pipfile.lock

@@ -22,6 +22,11 @@
 preload_process_types = ['extract', 'load']
 preload_system_keys = [{'key': 'version', 'value': '0.1.0'}]
 
+relational_stores = ['postgresql', 'mysql', 'oracle', 'mssql', 'snowflake']
+nonrelational_stores = []
+
+supported_data_stores = relational_stores + nonrelational_stores
+
 
 class DataStore:
 
@@ -362,18 +367,15 @@ def verify_and_connect_to_data_store(self):
 
             raise Exception(errors)
 
-        relational_stores = ['postgresql', 'mysql', 'snowflake']
-        nonrelational_stores = []
-
-        supported_data_stores = relational_stores + nonrelational_stores
-
         if data_store_type in supported_data_stores:
             engine = ''
             meta = ''
             session = ''
 
             if data_store_type in relational_stores:
-                if data_store_type == 'postgresql' or data_store_type == 'snowflake':
+                if data_store_type == 'postgresql' \
+                        or data_store_type == 'oracle'\
+                        or data_store_type == 'snowflake':
 
                     engine = create_engine(data_store_type + '://' + data_store_username + ':' + data_store_password
                                            + '@' + data_store_host + '/' + data_store_name)
@@ -382,6 +384,10 @@ def verify_and_connect_to_data_store(self):
 
                     engine = create_engine('mysql+pymysql://' + data_store_username + ':' + data_store_password + '@'
                                            + data_store_host + '/' + data_store_name)
+                elif data_store_type == 'mssql':
+
+                    engine = create_engine('mssql+pymssql://' + data_store_username + ':' + data_store_password + '@'
+                                           + data_store_host + '/' + data_store_name)
 
                 else:
                     self.logger.error("Data store type valid but not configured.")

process_tracker/data_store.py

@@ -19,7 +19,13 @@
     install_requires=[
         'boto3 >= 1.9.150',
         'Click >= 7.0',
+        'cs-oracle >= 7.1.3'
         'google-compute-engine >= 2.8.13',
+        'psycopg2-binary >= 2.8.2',
+        'pymysql >= 0.9.3',
+        'pymssql >= 2.1.4',
+        'python-dateutil >= 2.8.0',
+        'snowflake-sqlalchemy >= 1.1.13'
         'sqlalchemy >= 1.3.3',
         'sqlalchemy-utils >= 0.33.11',
         'pymysql >= 0.9.3',
@@ -29,9 +35,9 @@
     extras_requires={
         'dev': [
             'coverage >= "4.0.3',
+            'coveralls >= 1.7.0',
             'moto >= 1.3.8'
             'python-coveralls >= 2.9.1',
-            'coveralls >= 1.7.0',
         ]
     },
     entry_points = {