Merge pull request #44 from OpenDataAlex/process_tracker_python-23

Process tracker python 23

Author: OpenDataAlex

.travis.yml

@@ -21,8 +21,8 @@ before_script:
 - if [[ "$DB" == "mysql" ]]; then mysql -u root < dbscripts/mysql_process_tracker.sql; fi
 - if [[ "$DB" == "mysql" ]]; then mysql -u root < dbscripts/mysql_process_tracker_defaults.sql; fi
 env:
-- DB=postgres
-- DB=mysql
+- DB=postgres TRAVIS="true"
+- DB=mysql TRAVIS="true"
 deploy:
   - provider: releases
     api_key:

process_tracker/cli.py

@@ -20,25 +20,31 @@ def main():
     """
 
 
-# @main.command()
-# @click.option('-o', '--overwrite', default=False, help='Wipe out the current data store and rebuild'
-#                                                        ', starting from fresh.')
-# def setup(overwrite):
-#     """
-#     Initialize ProcessTracker's data store with user provided input.  If already in place, do nothing unless overwrite
-#     set to True.
-#     :return:
-#     """
-#     click.echo('Attempting to initialize data store...')
-#     data_store.initialize_data_store(overwrite=overwrite)
-
-
-# @main.command()
-# def upgrade():
-#     """
-#     Upgrade ProcessTracker if data store on previous version.
-#     :return:
-#     """
+@main.command()
+@click.option(
+    "-o",
+    "--overwrite",
+    default=False,
+    help="Wipe out the current data store and rebuild" ", starting from fresh.",
+)
+def setup(overwrite=False):
+    """
+    Initialize ProcessTracker's data store with user provided input.  If already in place, do nothing unless overwrite
+    set to True.
+    :return:
+    """
+    click.echo("Attempting to initialize data store...")
+    data_store.initialize_data_store(overwrite=overwrite)
+
+
+@main.command()
+def upgrade():
+    """
+    Upgrade ProcessTracker if data store on previous version.
+    :return:
+    """
+    click.echo("Verifying version installed on data store vs package...")
+    data_store.determine_versions()
 
 
 @main.command()

process_tracker/data_store.py

@@ -34,7 +34,7 @@
 ]
 preload_process_status_types = ["running", "completed", "failed"]
 preload_process_types = ["extract", "load"]
-preload_system_keys = [{"key": "version", "value": "0.2.0"}]
+preload_system_keys = [{"version", "0.2.0"}]
 
 supported_data_stores = ["postgresql", "mysql", "oracle", "mssql", "snowflake"]
 
@@ -61,6 +61,31 @@ def __init__(self, config_location=None):
         self.data_store_port = data_store["data_store_port"]
         self.data_store_name = data_store["data_store_name"]
 
+    def delete_data_store(self):
+        """
+        Initializes data store deletion, including wiping of all data within.
+        :return:
+        """
+
+        self.logger.warn("ALERT - DATA STORE TO BE OVERWRITTEN - ALL DATA WILL BE LOST")
+
+        for table in reversed(Base.metadata.sorted_tables):
+            try:
+                self.logger.info("Table will be deleted: %s" % table)
+                table.drop(self.engine)
+            except Exception:
+                self.logger.error(
+                    "Table %s unable to be deleted.  Does it exist?" % table
+                )
+
+    def determine_versions(self):
+        """
+        Find the data store version and the package version and return them.
+        :return:
+        """
+
+        self.session.query(System.system_value).filter(System.system_key == "version")
+
     def get_or_create_item(self, model, create=True, **kwargs):
         """
         Testing if an entity instance exists or not.  If does, return entity key.  If not, create entity instance
@@ -110,63 +135,50 @@ def initialize_data_store(self, overwrite=False):
         :type overwrite: bool
         """
         self.logger.info("Attempting to initialize Process Tracker data store.")
-        if overwrite:
-            self.logger.warn(
-                "ALERT - DATA STORE TO BE OVERWRITTEN - ALL DATA WILL BE LOST"
-            )
-
-            for table in Base.metadata.table_names():
-                self.logger.info("Table will be deleted: %s" % table)
-                table.drop(self.engine)
 
-            version = None
-        else:
-
-            self.logger.debug("Obtaining system version, if exists.")
-            version = (
-                self.session.query(System)
-                .filter(System.system_key == "version")
-                .first()
+        if overwrite:
+            self.delete_data_store()
+
+        self.logger.info("Data store initialization beginning.  Creating data store.")
+        for table in Base.metadata.sorted_tables:
+            try:
+                self.logger.info("Table will be created: %s" % table)
+                table.create(self.engine)
+            except Exception:
+                self.logger.error("Object %s already exists?" % table)
+
+        self.logger.info("Setting up application defaults.")
+
+        self.logger.info("Adding error types...")
+        for error_type in preload_error_types:
+            self.logger.info("Adding %s" % error_type)
+            self.get_or_create_item(model=ErrorType, error_type_name=error_type)
+
+        self.logger.info("Adding extract status types...")
+        for extract_status_type in preload_extract_status_types:
+            self.logger.info("Adding %s" % extract_status_type)
+            self.get_or_create_item(
+                ExtractStatus, extract_status_name=extract_status_type
             )
 
-        if version is None:
-
-            self.logger.info(
-                "Data store initialization beginning.  Creating data store."
+        self.logger.info("Adding process status types...")
+        for process_status_type in preload_process_status_types:
+            self.logger.info("Adding %s" % process_status_type)
+            self.get_or_create_item(
+                model=ProcessStatus, process_status_name=process_status_type
             )
-            Base.metadata.create_all(self.engine)
-
-            self.logger.info("Setting up application defaults.")
-
-            self.logger.info("Adding error types...")
-            for error_type in preload_error_types:
-                self.logger.info("Adding %s" % error_type)
-                self.session.add(ErrorType(error_type_name=error_type))
-
-            self.logger.info("Adding extract status types...")
-            for extract_status_type in preload_extract_status_types:
-                self.logger.info("Adding %s" % extract_status_type)
-                self.session.add(ExtractStatus(extract_status_name=extract_status_type))
 
-            self.logger.info("Adding process status types...")
-            for process_status_type in preload_process_status_types:
-                self.logger.info("Adding %s" % process_status_type)
-                self.session.add(ProcessStatus(process_status_name=process_status_type))
+        self.logger.info("Adding process types...")
+        for process_type in preload_process_types:
+            self.logger.info("Adding %s" % process_type)
+            self.get_or_create_item(model=ProcessType, process_type_name=process_type)
 
-            self.logger.info("Adding process types...")
-            for process_type in preload_process_types:
-                self.logger.info("Adding %s" % process_type)
-                self.session.add(ProcessType(process_type_name=process_type))
+        self.logger.info("Adding system keys...")
+        for key, value in preload_system_keys:
+            self.logger.info("Adding %s" % key)
+            self.get_or_create_item(model=System, system_key=key, system_value=value)
 
-            self.logger.info("Adding system keys...")
-            for system_key, value in preload_system_keys:
-                self.logger.info("Adding %s" % system_key)
-                self.session.add(System(system_key=system_key, system_value=value))
-
-            self.session.commit()
-        else:
-            self.logger.error("It appears the system has already been setup.")
-            ClickException("It appears the system has already been setup.").show()
+        self.session.commit()
 
         self.logger.debug("Finished the initialization check.")
 

process_tracker/models/actor.py

@@ -8,8 +8,13 @@
 class Actor(Base):
 
     __tablename__ = "actor_lkup"
+    __table_args__ = {"schema": "process_tracker"}
 
-    actor_id = Column(Integer, Sequence("actor_lkup_actor_id_seq"), primary_key=True)
+    actor_id = Column(
+        Integer,
+        Sequence("actor_lkup_actor_id_seq", schema="process_tracker"),
+        primary_key=True,
+    )
     actor_name = Column(String(250), nullable=False, unique=True)
 
     def __repr__(self):

process_tracker/models/extract.py

@@ -13,9 +13,12 @@
 class ExtractStatus(Base):
 
     __tablename__ = "extract_status_lkup"
+    __table_args__ = {"schema": "process_tracker"}
 
     extract_status_id = Column(
-        Integer, Sequence("extract_status_lkup_extract_status_id_seq"), primary_key=True
+        Integer,
+        Sequence("extract_status_lkup_extract_status_id_seq", schema="process_tracker"),
+        primary_key=True,
     )
     extract_status_name = Column(String(75), nullable=False, unique=True)
 
@@ -32,14 +35,19 @@ def __repr__(self):
 class Extract(Base):
 
     __tablename__ = "extract_tracking"
+    __table_args__ = {"schema": "process_tracker"}
 
     extract_id = Column(
-        Integer, Sequence("extract_tracking_extract_id_seq"), primary_key=True
+        Integer,
+        Sequence("extract_tracking_extract_id_seq", schema="process_tracker"),
+        primary_key=True,
     )
     extract_filename = Column(String(750), nullable=False, unique=True)
-    extract_location_id = Column(Integer, ForeignKey("location_lkup.location_id"))
+    extract_location_id = Column(
+        Integer, ForeignKey("process_tracker.location_lkup.location_id")
+    )
     extract_status_id = Column(
-        Integer, ForeignKey("extract_status_lkup.extract_status_id")
+        Integer, ForeignKey("process_tracker.extract_status_lkup.extract_status_id")
     )
     extract_registration_date_time = Column(
         DateTime, nullable=False, default=datetime.now()
@@ -71,12 +79,17 @@ def full_filepath(self):
 class ExtractDependency(Base):
 
     __tablename__ = "extract_dependency"
+    __table_args__ = {"schema": "process_tracker"}
 
     parent_extract_id = Column(
-        Integer, ForeignKey("extract_tracking.extract_id"), primary_key=True
+        Integer,
+        ForeignKey("process_tracker.extract_tracking.extract_id"),
+        primary_key=True,
     )
     child_extract_id = Column(
-        Integer, ForeignKey("extract_tracking.extract_id"), primary_key=True
+        Integer,
+        ForeignKey("process_tracker.extract_tracking.extract_id"),
+        primary_key=True,
     )
 
     child_extract = relationship("Extract", foreign_keys=[child_extract_id])
@@ -93,15 +106,20 @@ def __repr__(self):
 class ExtractProcess(Base):
 
     __tablename__ = "extract_process_tracking"
+    __table_args__ = {"schema": "process_tracker"}
 
     extract_tracking_id = Column(
-        Integer, ForeignKey("extract_tracking.extract_id"), primary_key=True
+        Integer,
+        ForeignKey("process_tracker.extract_tracking.extract_id"),
+        primary_key=True,
     )
     process_tracking_id = Column(
-        Integer, ForeignKey("process_tracking.process_tracking_id"), primary_key=True
+        Integer,
+        ForeignKey("process_tracker.process_tracking.process_tracking_id"),
+        primary_key=True,
     )
     extract_process_status_id = Column(
-        Integer, ForeignKey("extract_status_lkup.extract_status_id")
+        Integer, ForeignKey("process_tracker.extract_status_lkup.extract_status_id")
     )
     extract_process_event_date_time = Column(
         DateTime, nullable=False, default=datetime.now()
@@ -124,9 +142,12 @@ def __repr__(self):
 class LocationType(Base):
 
     __tablename__ = "location_type_lkup"
+    __table_args__ = {"schema": "process_tracker"}
 
     location_type_id = Column(
-        Integer, Sequence("location_type_lkup_location_type_id_seq"), primary_key=True
+        Integer,
+        Sequence("location_type_lkup_location_type_id_seq", schema="process_tracker"),
+        primary_key=True,
     )
     location_type_name = Column(String(25), unique=True, nullable=False)
 
@@ -143,13 +164,18 @@ def __repr__(self):
 class Location(Base):
 
     __tablename__ = "location_lkup"
+    __table_args__ = {"schema": "process_tracker"}
 
     location_id = Column(
-        Integer, Sequence("location_lkup_location_id_seq"), primary_key=True
+        Integer,
+        Sequence("location_lkup_location_id_seq", schema="process_tracker"),
+        primary_key=True,
     )
     location_name = Column(String(750), nullable=False, unique=True)
     location_path = Column(String(750), nullable=False, unique=True)
-    location_type = Column(Integer, ForeignKey("location_type_lkup.location_type_id"))
+    location_type = Column(
+        Integer, ForeignKey("process_tracker.location_type_lkup.location_type_id")
+    )
 
     extracts = relationship("Extract")