process_tracker_python-30 Extract Lookups should return Extract objects

All the extract file finders now return extract objects so they can be
modified as they are used.  A helper method was also added that provides
the full filename of a given extract (called full_filepath).

Also realized that I missed adding string representations of a bunch of
the data models.

Closes #30

Author: OpenDataAlex

process_tracker/models/actor.py

@@ -11,3 +11,7 @@ class Actor(Base):
 
     actor_id = Column(Integer, Sequence('actor_lkup_actor_id_seq'), primary_key=True)
     actor_name = Column(String(250), nullable=False, unique=True)
+
+    def __repr__(self):
+        return "<Actor id=%s, name=%s>" % (self.actor_id
+                                          , self.actor_name)

process_tracker/models/extract.py

@@ -2,6 +2,7 @@
 # Models for Extract (Data) entities
 
 from datetime import datetime
+from os.path import join
 
 from sqlalchemy import Column, DateTime, ForeignKey, Integer, Sequence, String
 from sqlalchemy.orm import relationship
@@ -18,6 +19,11 @@ class ExtractStatus(Base):
 
     extracts = relationship("ExtractProcess")
 
+    def __repr__(self):
+
+        return "<Extract Status id=%s, name=%s>" % (self.extract_status_id
+                                                    , self.extract_status_name)
+
 
 class Extract(Base):
 
@@ -32,6 +38,17 @@ class Extract(Base):
     extract_process = relationship("ExtractProcess", back_populates='process_extracts')
     locations = relationship("Location", foreign_keys=[extract_location_id])
 
+    def __repr__(self):
+
+        return "<Extract id=%s, filename=%s, location=%s, status=%s>" % (self.extract_id
+                                                                         , self.extract_filename
+                                                                         , self.extract_location_id
+                                                                         , self.extract_status_id)
+
+    def full_filepath(self):
+
+        return join(self.locations.location_path, self.extract_filename)
+
 
 class ExtractProcess(Base):
 
@@ -45,6 +62,12 @@ class ExtractProcess(Base):
     process_extracts = relationship('Extract', foreign_keys=[extract_tracking_id])
     extract_processes = relationship('ProcessTracking', foreign_keys=[process_tracking_id])
 
+    def __repr__(self):
+
+        return "<ExtractProcess extract=%s, process_run=%s, extract_status=%s>" % (self.extract_tracking_id
+                                                                                   , self.process_tracking_id
+                                                                                   , self.extract_process_status_id)
+
 
 class LocationType(Base):
 
@@ -54,7 +77,12 @@ class LocationType(Base):
     location_type_name = Column(String(25), unique=True, nullable=False)
 
     locations = relationship('Location', back_populates='location_types')
-    
+
+    def __repr__(self):
+
+        return "<LocationType id=%s, name=%s>" % (self.location_type_id
+                                                  , self.location_type_name)
+
 
 class Location(Base):
 
@@ -68,3 +96,9 @@ class Location(Base):
     extracts = relationship("Extract")
 
     location_types = relationship('LocationType', foreign_keys=[location_type])
+
+    def __repr__(self):
+
+        return "<Location id=%s, name=%s, type=%s>" % (self.location_id
+                                                       , self.location_name
+                                                       , self.location_path)

process_tracker/process_tracker.py

@@ -4,7 +4,6 @@
 from datetime import datetime
 import logging
 import os
-from os.path import join
 
 from sqlalchemy.orm import aliased
 
@@ -110,62 +109,54 @@ def find_ready_extracts_by_filename(self, filename):
         :param filename:
         :return:
         """
-        extract_files = []
 
-        process_files = self.session.query(Extract.extract_filename, Location.location_path)\
-                                           .join(Location)\
+        process_files = self.session.query(Extract)\
                                            .join(ExtractStatus)\
                                            .filter(Extract.extract_filename.like("%" + filename + "%"))\
                                            .filter(ExtractStatus.extract_status_name == 'ready') \
                                            .order_by(Extract.extract_registration_date_time)\
-                                           .order_by(Extract.extract_id)
+                                           .order_by(Extract.extract_id)\
+                                           .all()
 
-        for record in process_files:
-            extract_files.append(join(record.location_path, record.extract_filename))
-
-        return extract_files
+        return process_files
 
     def find_ready_extracts_by_location(self, location):
         """
         For the given location name, find all matching extracts that are ready for processing
         :param location:
         :return:
         """
-        extract_files = []
 
-        process_files = self.session.query(Extract.extract_filename, Location.location_path)\
+        process_files = self.session.query(Extract)\
                                .join(Location)\
                                .join(ExtractStatus)\
                                .filter(ExtractStatus.extract_status_name == 'ready')\
                                .filter(Location.location_name == location) \
-                               .order_by(Extract.extract_registration_date_time)
-
-        for record in process_files:
-            extract_files.append(join(record.location_path, record.extract_filename))
+                               .order_by(Extract.extract_registration_date_time)\
+                               .all()
 
-        return extract_files
+        return process_files
 
     def find_ready_extracts_by_process(self, extract_process_name):
         """
         For the given named process, find the extracts that are ready for processing.
         :return: List of OS specific filepaths with filenames.
         """
-        extract_files = []
 
-        process_files = self.session.query(Extract.extract_filename, Location.location_path) \
+        process_files = self.session.query(Extract) \
             .join(ExtractStatus, Extract.extract_status_id == ExtractStatus.extract_status_id) \
             .join(Location, Extract.extract_location_id == Location.location_id) \
             .join(ExtractProcess, Extract.extract_id == ExtractProcess.extract_tracking_id) \
             .join(ProcessTracking) \
             .join(Process) \
             .filter(Process.process_name == extract_process_name
                     , ExtractStatus.extract_status_name == 'ready') \
-            .order_by(Extract.extract_registration_date_time)
+            .order_by(Extract.extract_registration_date_time)\
+            .all()
 
-        for record in process_files:
-            extract_files.append(join(record.location_path, record.extract_filename))
+        self.logger.info('Returning extract files by process.')
 
-        return extract_files
+        return process_files
 
     def get_latest_tracking_record(self, process):
         """

tests/test_process_tracker.py

@@ -129,8 +129,9 @@ def test_find_ready_extracts_by_filename_full(self):
         expected_result = ['/home/test/extract_dir/test_extract_filename2.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_filename('test_extract_filename2.csv')
+        given_result = [record.full_filepath() for record in given_result]
 
-        self.assertEqual(expected_result, given_result)
+        self.assertCountEqual(expected_result, given_result)
 
     def test_find_ready_extracts_by_filename_partial(self):
         """
@@ -161,8 +162,9 @@ def test_find_ready_extracts_by_filename_partial(self):
                            , '/home/test/extract_dir/test_extract_filename3-2.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_filename('test_extract_filename')
+        given_result = [record.full_filepath() for record in given_result]
 
-        self.assertEqual(expected_result, given_result)
+        self.assertCountEqual(expected_result, given_result)
 
     def test_find_ready_extracts_by_filename_partial_not_descending(self):
         """
@@ -193,6 +195,7 @@ def test_find_ready_extracts_by_filename_partial_not_descending(self):
                            , '/home/test/extract_dir/test_extract_filename3-1.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_filename('test_extract_filename')
+        given_result = [record.full_filepath() for record in given_result]
 
         self.assertNotEqual(expected_result, given_result)
 
@@ -225,8 +228,9 @@ def test_find_ready_extracts_by_location(self):
                            , '/home/test/extract_dir/test_extract_filename4-2.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_location('Test Location')
+        given_result = [record.full_filepath() for record in given_result]
 
-        self.assertEqual(expected_result, given_result)
+        self.assertCountEqual(expected_result, given_result)
 
     def test_find_ready_extracts_by_location_not_descending(self):
         """
@@ -257,6 +261,7 @@ def test_find_ready_extracts_by_location_not_descending(self):
                            , '/home/test/extract_dir/test_extract_filename4-1.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_location('Test Location')
+        given_result = [record.full_filepath() for record in given_result]
 
         self.assertNotEqual(expected_result, given_result)
 
@@ -288,8 +293,9 @@ def test_find_ready_extracts_by_process(self):
                            , '/home/test/extract_dir/test_extract_filename5-2.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_process('Testing Process Tracking Initialization')
+        given_result = [record.full_filepath() for record in given_result]
 
-        self.assertEqual(sorted(expected_result), sorted(given_result))
+        self.assertCountEqual(expected_result, given_result)
 
     def test_find_ready_extracts_by_process_not_descending(self):
         """
@@ -323,6 +329,7 @@ def test_find_ready_extracts_by_process_not_descending(self):
             , '/home/test/extract_dir/test_extract_filename5-1.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_process('Testing Process Tracking Initialization')
+        given_result = [record.full_filepath() for record in given_result]
 
         self.assertNotEqual(expected_result, given_result)