process_tracker_python-23 Data store initialization thru CLI

✨ CLI can now initialize data store (or drop and recreate)

Found the issue to why drop_all and create_all were not working.  Needed
to include the schema for objects.

Working on #23

Author: OpenDataAlex

process_tracker/cli.py

@@ -20,17 +20,21 @@ def main():
     """
 
 
-# @main.command()
-# @click.option('-o', '--overwrite', default=False, help='Wipe out the current data store and rebuild'
-#                                                        ', starting from fresh.')
-# def setup(overwrite):
-#     """
-#     Initialize ProcessTracker's data store with user provided input.  If already in place, do nothing unless overwrite
-#     set to True.
-#     :return:
-#     """
-#     click.echo('Attempting to initialize data store...')
-#     data_store.initialize_data_store(overwrite=overwrite)
+@main.command()
+@click.option(
+    "-o",
+    "--overwrite",
+    default=False,
+    help="Wipe out the current data store and rebuild" ", starting from fresh.",
+)
+def setup(overwrite=False):
+    """
+    Initialize ProcessTracker's data store with user provided input.  If already in place, do nothing unless overwrite
+    set to True.
+    :return:
+    """
+    click.echo("Attempting to initialize data store...")
+    data_store.initialize_data_store(overwrite=overwrite)
 
 
 # @main.command()

process_tracker/data_store.py

@@ -34,7 +34,7 @@
 ]
 preload_process_status_types = ["running", "completed", "failed"]
 preload_process_types = ["extract", "load"]
-preload_system_keys = [{"key": "version", "value": "0.2.0"}]
+preload_system_keys = [{"version", "0.2.0"}]
 
 supported_data_stores = ["postgresql", "mysql", "oracle", "mssql", "snowflake"]
 
@@ -110,63 +110,54 @@ def initialize_data_store(self, overwrite=False):
         :type overwrite: bool
         """
         self.logger.info("Attempting to initialize Process Tracker data store.")
+
         if overwrite:
             self.logger.warn(
                 "ALERT - DATA STORE TO BE OVERWRITTEN - ALL DATA WILL BE LOST"
             )
+            self.session.commit()
+            Base.metadata.drop_all(self.engine)
 
-            for table in Base.metadata.table_names():
-                self.logger.info("Table will be deleted: %s" % table)
-                table.drop(self.engine)
+        self.session.commit()
+        self.logger.info("Data store initialization beginning.  Creating data store.")
+        Base.metadata.create_all(self.engine)
 
-            version = None
-        else:
+        # for table in Base.metadata.sorted_tables:
+        #     self.logger.info("Table will be created: %s" % table)
+        #     table.create(self.engine)
 
-            self.logger.debug("Obtaining system version, if exists.")
-            version = (
-                self.session.query(System)
-                .filter(System.system_key == "version")
-                .first()
-            )
+        self.logger.info("Setting up application defaults.")
 
-        if version is None:
+        self.logger.info("Adding error types...")
+        for error_type in preload_error_types:
+            self.logger.info("Adding %s" % error_type)
+            self.get_or_create_item(model=ErrorType, error_type_name=error_type)
 
-            self.logger.info(
-                "Data store initialization beginning.  Creating data store."
+        self.logger.info("Adding extract status types...")
+        for extract_status_type in preload_extract_status_types:
+            self.logger.info("Adding %s" % extract_status_type)
+            self.get_or_create_item(
+                ExtractStatus, extract_status_name=extract_status_type
             )
-            Base.metadata.create_all(self.engine)
-
-            self.logger.info("Setting up application defaults.")
 
-            self.logger.info("Adding error types...")
-            for error_type in preload_error_types:
-                self.logger.info("Adding %s" % error_type)
-                self.session.add(ErrorType(error_type_name=error_type))
-
-            self.logger.info("Adding extract status types...")
-            for extract_status_type in preload_extract_status_types:
-                self.logger.info("Adding %s" % extract_status_type)
-                self.session.add(ExtractStatus(extract_status_name=extract_status_type))
-
-            self.logger.info("Adding process status types...")
-            for process_status_type in preload_process_status_types:
-                self.logger.info("Adding %s" % process_status_type)
-                self.session.add(ProcessStatus(process_status_name=process_status_type))
+        self.logger.info("Adding process status types...")
+        for process_status_type in preload_process_status_types:
+            self.logger.info("Adding %s" % process_status_type)
+            self.get_or_create_item(
+                model=ProcessStatus, process_status_name=process_status_type
+            )
 
-            self.logger.info("Adding process types...")
-            for process_type in preload_process_types:
-                self.logger.info("Adding %s" % process_type)
-                self.session.add(ProcessType(process_type_name=process_type))
+        self.logger.info("Adding process types...")
+        for process_type in preload_process_types:
+            self.logger.info("Adding %s" % process_type)
+            self.get_or_create_item(model=ProcessType, process_type_name=process_type)
 
-            self.logger.info("Adding system keys...")
-            for system_key, value in preload_system_keys:
-                self.logger.info("Adding %s" % system_key)
-                self.session.add(System(system_key=system_key, system_value=value))
+        self.logger.info("Adding system keys...")
+        for key, value in preload_system_keys:
+            self.logger.info("Adding %s" % key)
+            self.get_or_create_item(model=System, system_key=key, system_value=value)
 
-            self.session.commit()
-        else:
-            self.logger.error("It appears the system has already been setup.")
-            ClickException("It appears the system has already been setup.").show()
+        self.session.commit()
 
         self.logger.debug("Finished the initialization check.")
 

process_tracker/models/actor.py

@@ -8,8 +8,13 @@
 class Actor(Base):
 
     __tablename__ = "actor_lkup"
+    __table_args__ = {"schema": "process_tracker"}
 
-    actor_id = Column(Integer, Sequence("actor_lkup_actor_id_seq"), primary_key=True)
+    actor_id = Column(
+        Integer,
+        Sequence("actor_lkup_actor_id_seq", schema="process_tracker"),
+        primary_key=True,
+    )
     actor_name = Column(String(250), nullable=False, unique=True)
 
     def __repr__(self):

process_tracker/models/extract.py

@@ -13,9 +13,12 @@
 class ExtractStatus(Base):
 
     __tablename__ = "extract_status_lkup"
+    __table_args__ = {"schema": "process_tracker"}
 
     extract_status_id = Column(
-        Integer, Sequence("extract_status_lkup_extract_status_id_seq"), primary_key=True
+        Integer,
+        Sequence("extract_status_lkup_extract_status_id_seq", schema="process_tracker"),
+        primary_key=True,
     )
     extract_status_name = Column(String(75), nullable=False, unique=True)
 
@@ -32,14 +35,19 @@ def __repr__(self):
 class Extract(Base):
 
     __tablename__ = "extract_tracking"
+    __table_args__ = {"schema": "process_tracker"}
 
     extract_id = Column(
-        Integer, Sequence("extract_tracking_extract_id_seq"), primary_key=True
+        Integer,
+        Sequence("extract_tracking_extract_id_seq", schema="process_tracker"),
+        primary_key=True,
     )
     extract_filename = Column(String(750), nullable=False, unique=True)
-    extract_location_id = Column(Integer, ForeignKey("location_lkup.location_id"))
+    extract_location_id = Column(
+        Integer, ForeignKey("process_tracker.location_lkup.location_id")
+    )
     extract_status_id = Column(
-        Integer, ForeignKey("extract_status_lkup.extract_status_id")
+        Integer, ForeignKey("process_tracker.extract_status_lkup.extract_status_id")
     )
     extract_registration_date_time = Column(
         DateTime, nullable=False, default=datetime.now()
@@ -71,12 +79,17 @@ def full_filepath(self):
 class ExtractDependency(Base):
 
     __tablename__ = "extract_dependency"
+    __table_args__ = {"schema": "process_tracker"}
 
     parent_extract_id = Column(
-        Integer, ForeignKey("extract_tracking.extract_id"), primary_key=True
+        Integer,
+        ForeignKey("process_tracker.extract_tracking.extract_id"),
+        primary_key=True,
     )
     child_extract_id = Column(
-        Integer, ForeignKey("extract_tracking.extract_id"), primary_key=True
+        Integer,
+        ForeignKey("process_tracker.extract_tracking.extract_id"),
+        primary_key=True,
     )
 
     child_extract = relationship("Extract", foreign_keys=[child_extract_id])
@@ -93,15 +106,20 @@ def __repr__(self):
 class ExtractProcess(Base):
 
     __tablename__ = "extract_process_tracking"
+    __table_args__ = {"schema": "process_tracker"}
 
     extract_tracking_id = Column(
-        Integer, ForeignKey("extract_tracking.extract_id"), primary_key=True
+        Integer,
+        ForeignKey("process_tracker.extract_tracking.extract_id"),
+        primary_key=True,
     )
     process_tracking_id = Column(
-        Integer, ForeignKey("process_tracking.process_tracking_id"), primary_key=True
+        Integer,
+        ForeignKey("process_tracker.process_tracking.process_tracking_id"),
+        primary_key=True,
     )
     extract_process_status_id = Column(
-        Integer, ForeignKey("extract_status_lkup.extract_status_id")
+        Integer, ForeignKey("process_tracker.extract_status_lkup.extract_status_id")
     )
     extract_process_event_date_time = Column(
         DateTime, nullable=False, default=datetime.now()
@@ -124,9 +142,12 @@ def __repr__(self):
 class LocationType(Base):
 
     __tablename__ = "location_type_lkup"
+    __table_args__ = {"schema": "process_tracker"}
 
     location_type_id = Column(
-        Integer, Sequence("location_type_lkup_location_type_id_seq"), primary_key=True
+        Integer,
+        Sequence("location_type_lkup_location_type_id_seq", schema="process_tracker"),
+        primary_key=True,
     )
     location_type_name = Column(String(25), unique=True, nullable=False)
 
@@ -143,13 +164,18 @@ def __repr__(self):
 class Location(Base):
 
     __tablename__ = "location_lkup"
+    __table_args__ = {"schema": "process_tracker"}
 
     location_id = Column(
-        Integer, Sequence("location_lkup_location_id_seq"), primary_key=True
+        Integer,
+        Sequence("location_lkup_location_id_seq", schema="process_tracker"),
+        primary_key=True,
     )
     location_name = Column(String(750), nullable=False, unique=True)
     location_path = Column(String(750), nullable=False, unique=True)
-    location_type = Column(Integer, ForeignKey("location_type_lkup.location_type_id"))
+    location_type = Column(
+        Integer, ForeignKey("process_tracker.location_type_lkup.location_type_id")
+    )
 
     extracts = relationship("Extract")