Added Extract Dependency Tracking

✨ Ability to have extract interdependency

ExtractTracker now has dependency tracking.

Working: #14

Author: OpenDataAlex

Pipfile

@@ -4,10 +4,11 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
+black = "*"
 coverage="*"
+coveralls="*"
 moto="*"
 python-coveralls="*"
-coveralls="*"
 
 [packages]
 boto3="*"
@@ -21,7 +22,6 @@ snowflake-sqlalchemy = "*"
 pymysql = "*"
 cx-oracle = "*"
 pymssql = "*"
-black = "*"
 
 [requires]
 python_version = "3.7"

process_tracker/extract_tracker.py

@@ -182,6 +182,9 @@ def extract_dependency_check(self):
                 "created, or are in the process of loading."
             )
 
+        else:
+            return False
+
     def get_extract_status_types(self):
         """
         Get list of process status types and return dictionary.

tests/test_extract_tracker.py

@@ -12,7 +12,7 @@
 )
 
 from process_tracker.data_store import DataStore
-from process_tracker.extract_tracker import ExtractTracker
+from process_tracker.extract_tracker import ExtractDependency, ExtractTracker
 from process_tracker.process_tracker import ErrorTracking, ProcessTracker
 
 
@@ -64,6 +64,7 @@ def tearDown(self):
         Need to clean up tables to return them to pristine state for other tests.
         :return:
         """
+        self.session.query(ExtractDependency).delete()
         self.session.query(ExtractProcess).delete()
         self.session.query(Extract).delete()
         self.session.query(Location).delete()
@@ -192,13 +193,56 @@ def test_extract_dependency_check(self):
         is loaded.
         :return:
         """
+        dependent_extract = ExtractTracker(
+            process_run=self.process_run,
+            filename="Dependent File.csv",
+            location_name="Test Location",
+            location_path="/home/test/extract_dir",
+        )
+        dependency = ExtractDependency(
+            child_extract_id=dependent_extract.extract.extract_id,
+            parent_extract_id=self.extract.extract.extract_id,
+        )
+
+        self.session.add(dependency)
+        self.session.commit()
+        self.extract.change_extract_status("loaded")
+
+        given_result = dependent_extract.extract_dependency_check()
+
+        expected_result = False
+
+        self.assertEqual(expected_result, given_result)
 
     def test_extract_dependency_check_blocked(self):
         """
         Testing that if a dependency is in a state that stops an extract from being loaded, then the extract triggers an
         error blocking the file from being processed.
         :return:
         """
+        dependent_extract = ExtractTracker(
+            process_run=self.process_run,
+            filename="Dependent File.csv",
+            location_name="Test Location",
+            location_path="/home/test/extract_dir",
+        )
+        dependency = ExtractDependency(
+            child_extract_id=dependent_extract.extract.extract_id,
+            parent_extract_id=self.extract.extract.extract_id,
+        )
+
+        self.session.add(dependency)
+        self.session.commit()
+
+        self.extract.change_extract_status("loading")
+
+        with self.assertRaises(Exception) as context:
+            dependent_extract.extract_dependency_check()
+
+        return self.assertTrue(
+            "Extract files that this extract file is dependent on have not been loaded, are being "
+            "created, or are in the process of loading." in str(context.exception)
+        )
 
     def test_location_name_provided(self):
         """

tests/test_process_tracker.py

@@ -473,7 +473,7 @@ def test_register_extracts_by_location_local(self):
             ["test_local_dir_2.csv", "ready", "ready"],
         ]
 
-        self.assertEqual(expected_result, given_result)
+        self.assertCountEqual(expected_result, given_result)
 
     # def test_register_extracts_by_location_s3(self):
     #     """