Enforced extract record ascending order

🐛 Closes: #13
🏗️ Configured Travis-CI to publish on Github Releases

Resolved #13 where there was a chance where extracts could be returned
in an order other than registration date.  While this was unlikely, the order is now enforced in the queries.

Also tweaked travis-ci to now publish releases to both PyPi and GitHub.  This is based on git tags.  Will not test the publishing until more of the first release backlog is done.

Author: OpenDataAlex

.travis.yml

@@ -18,9 +18,16 @@ env:
   process_tracking_data_store_name=process_tracking process_tracking_data_store_type=postgresql
   process_tracking_data_store_password=Testing1! process_tracking_data_store_username=pt_admin
 deploy:
+  - provider: releases
+    api_key:
+      secure: T02NqddtE6qN/2ugvx/anZFzqmfnpWy2uCWcW4jg+pgiNI8GhRr0zh5Fwfe3aGnzAZLEgEMSgNWsM5CpniSbquioijqOEL/9xo8gZn6wV9VEjToR1/FEvGZm08T/Grc4rEYzjNgZu8eLUtC6Mg5l1nm6tGNEBE1oAKvflL77pnSGPL9enhmQupzbSQxNJ7rOuR7OgxuaFygP0Tne56Ca1LU6rAHmzqKJh6hI6eZARSikjLvgEHThcvZPc3gPfYMAc7i+dTx17Q3zQQGaHvySuLp+5aR3a+3xaFQPCmecLeiuX2yCiVVs9BOcrTeJOHivRYv6Cj9i60NUblvBlwOfJdo6uiArYTbVP/Aj8d1+8ruM5RdSeEyzlZlKarRN24CvH99OxU+LZGF5JgKV9UrTlytWl6ENOJcZq91LjejzkxGtKFp67abh8WPqqSGdcwA5VeI+QcHWDvc/3T/CdvljH4ivZRKeTCArAe4ce94WRvnJ6W0o3PoxrYxprhDFUyGUmrRJaLy/22qkkaifBQ9ljFgWNqeJtSGej87QhUVlfVnSS/6intr/sgd/vT0sDOU8RMTDxji3vkKKO1hfzBPKousqq48ddWGUxzAGwRoGULVbpwf8IIYApxlqotp40GpNOPJR6WrMUhHu/7jWGmvy66HPi4rQXKlaIWJLaH1f1yo=
+    file: ''
+    on:
+      repo: OpenDataAlex/process_tracker_python
   - provider: pypi
     user: opendataalex
     password:
       secure: jYe8ygGYuJox5f1vQvraIDNlR2Yk/+/afIXdsxaxRi/QN135rIHxRih24iYv8ZQewfW+w4CveEPz2pr0ew5NG8hm2MD5hQZI/A2QE4BC53Tr3xMAWSdPQgcI1rX4Ieu6BHTS3jet212yLuBk41GdASKztz+s/CVvVqT0A29U1b85OVzoVHz6OTgaM1m0pKeJE6Ho66r5O6J7ClZfEY4RJ8jROmrT7ufW3hEs/VQXwfAUWcjy6l5H5VYTLa/g94MgQ+0WmgcdUbB8s3c2q7n4RiaVEowZOLtBmRElDIPiLJC8JGAIFKcf5W8pzXK/ZhZhY3ppgmBvRVeyPdiAwjysionzl4vIlNxC28acI3HzvJbAdvwPsuMD+pnTo/LKRqqvGey2NSnMPmzGXxzqVUqQ5Wa1Yzdcqg8mbWHFRw9icMWKM84CnDBvqkmJcfnj5R4X6Rc77mr9jc5LMMBBblNtY7dQc2AooframujGA2mP1xABQ9vOSg89MdnlhHrgEK6m+Pz0S2hhlsGRw9yMF9L4Q1ie7xKeE49L3HKZSjKEgSMv9y9RbqnmW32F0UoKgWkCJE8DV9wbZe5Htn+lpwmXI0vu+cxnqsiZvG8wjm+klJGn5F84OFTN3zQOC/kIBXQXvutZYtgF0vofdCmKQzK9istl16Eur1r4NCbov3oM97o=
     on:
       tags: true
+  -

process_tracker/process_tracker.py

@@ -97,10 +97,11 @@ def find_ready_extracts_by_filename(self, filename):
         extract_files = []
 
         process_files = self.session.query(Extract.extract_filename, Location.location_path)\
-                               .join(Location)\
-                               .join(ExtractStatus)\
-                               .filter(Extract.extract_filename.like("%" + filename + "%"))\
-                               .filter(ExtractStatus.extract_status_name == 'ready')
+                                           .join(Location)\
+                                           .join(ExtractStatus)\
+                                           .filter(Extract.extract_filename.like("%" + filename + "%"))\
+                                           .filter(ExtractStatus.extract_status_name == 'ready') \
+                                           .order_by(Extract.extract_registration_date_time)
 
         for record in process_files:
             extract_files.append(join(record.location_path, record.extract_filename))
@@ -119,7 +120,8 @@ def find_ready_extracts_by_location(self, location):
                                .join(Location)\
                                .join(ExtractStatus)\
                                .filter(ExtractStatus.extract_status_name == 'ready')\
-                               .filter(Location.location_name == location)
+                               .filter(Location.location_name == location) \
+                               .order_by(Extract.extract_registration_date_time)
 
         for record in process_files:
             extract_files.append(join(record.location_path, record.extract_filename))
@@ -140,7 +142,8 @@ def find_ready_extracts_by_process(self, extract_process_name):
             .join(ProcessTracking) \
             .join(Process) \
             .filter(Process.process_name == extract_process_name
-                    , ExtractStatus.extract_status_name == 'ready')
+                    , ExtractStatus.extract_status_name == 'ready') \
+            .order_by(Extract.extract_registration_date_time)
 
         for record in process_files:
             extract_files.append(join(record.location_path, record.extract_filename))

tests/test_process_tracker.py

@@ -86,11 +86,17 @@ def test_find_ready_extracts_by_filename_full(self):
 
     def test_find_ready_extracts_by_filename_partial(self):
         """
-        Testing that for the given partial filename, find the extracts, provided they are in 'ready' state.
+        Testing that for the given partial filename, find the extracts, provided they are in 'ready' state.  Should return
+        them in ascending order by registration datetime.
         :return:
         """
         extract = ExtractTracker(process_run=self.process_tracker
-                       , filename='test_extract_filename3.csv'
+                       , filename='test_extract_filename3-1.csv'
+                       , location_name='Test Location'
+                       , location_path='/home/test/extract_dir')
+
+        extract2 = ExtractTracker(process_run=self.process_tracker
+                       , filename='test_extract_filename3-2.csv'
                        , location_name='Test Location'
                        , location_path='/home/test/extract_dir')
 
@@ -99,19 +105,62 @@ def test_find_ready_extracts_by_filename_partial(self):
         session = Session.object_session(extract.extract)
         session.commit()
 
-        expected_result = ['/home/test/extract_dir/test_extract_filename3.csv']
+        extract2.extract.extract_status_id = extract2.extract_status_ready
+        session = Session.object_session(extract2.extract)
+        session.commit()
+
+        expected_result = ['/home/test/extract_dir/test_extract_filename3-1.csv'
+                           , '/home/test/extract_dir/test_extract_filename3-2.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_filename('test_extract_filename')
 
         self.assertEqual(expected_result, given_result)
 
+    def test_find_ready_extracts_by_filename_partial_not_descending(self):
+        """
+        Testing that for the given partial filename, find the extracts, provided they are in 'ready' state.  Verifying
+        that records are NOT returned in descending order.
+        :return:
+        """
+        extract = ExtractTracker(process_run=self.process_tracker
+                       , filename='test_extract_filename3-1.csv'
+                       , location_name='Test Location'
+                       , location_path='/home/test/extract_dir')
+
+        extract2 = ExtractTracker(process_run=self.process_tracker
+                       , filename='test_extract_filename3-2.csv'
+                       , location_name='Test Location'
+                       , location_path='/home/test/extract_dir')
+
+        # Need to manually change the status, because this would normally be done while the process was processing data
+        extract.extract.extract_status_id = extract.extract_status_ready
+        session = Session.object_session(extract.extract)
+        session.commit()
+
+        extract2.extract.extract_status_id = extract2.extract_status_ready
+        session = Session.object_session(extract2.extract)
+        session.commit()
+
+        expected_result = ['/home/test/extract_dir/test_extract_filename3-2.csv'
+                           , '/home/test/extract_dir/test_extract_filename3-1.csv']
+
+        given_result = self.process_tracker.find_ready_extracts_by_filename('test_extract_filename')
+
+        self.assertNotEqual(expected_result, given_result)
+
     def test_find_ready_extracts_by_location(self):
         """
-        Testing that for the given location name, find the extracts, provided they are in 'ready' state.
+        Testing that for the given location name, find the extracts, provided they are in 'ready' state.  Should return
+        them in ascending order by registration datettime.
         :return:
         """
         extract = ExtractTracker(process_run=self.process_tracker
-                       , filename='test_extract_filename4.csv'
+                       , filename='test_extract_filename4-1.csv'
+                       , location_name='Test Location'
+                       , location_path='/home/test/extract_dir')
+
+        extract2 = ExtractTracker(process_run=self.process_tracker
+                       , filename='test_extract_filename4-2.csv'
                        , location_name='Test Location'
                        , location_path='/home/test/extract_dir')
 
@@ -120,19 +169,61 @@ def test_find_ready_extracts_by_location(self):
         session = Session.object_session(extract.extract)
         session.commit()
 
-        expected_result = ['/home/test/extract_dir/test_extract_filename4.csv']
+        extract2.extract.extract_status_id = extract2.extract_status_ready
+        session = Session.object_session(extract2.extract)
+        session.commit()
+
+        expected_result = ['/home/test/extract_dir/test_extract_filename4-1.csv'
+                           , '/home/test/extract_dir/test_extract_filename4-2.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_location('Test Location')
 
         self.assertEqual(expected_result, given_result)
 
+    def test_find_ready_extracts_by_location_not_descending(self):
+        """
+        Testing that for the given location name, find the extracts, provided they are in 'ready' state.  Verifying that
+        records NOT returned in descending order.
+        :return:
+        """
+        extract = ExtractTracker(process_run=self.process_tracker
+                       , filename='test_extract_filename4-1.csv'
+                       , location_name='Test Location'
+                       , location_path='/home/test/extract_dir')
+
+        extract2 = ExtractTracker(process_run=self.process_tracker
+                       , filename='test_extract_filename4-2.csv'
+                       , location_name='Test Location'
+                       , location_path='/home/test/extract_dir')
+
+        # Need to manually change the status, because this would normally be done while the process was processing data
+        extract.extract.extract_status_id = extract.extract_status_ready
+        session = Session.object_session(extract.extract)
+        session.commit()
+
+        extract2.extract.extract_status_id = extract2.extract_status_ready
+        session = Session.object_session(extract2.extract)
+        session.commit()
+
+        expected_result = ['/home/test/extract_dir/test_extract_filename4-2.csv'
+                           , '/home/test/extract_dir/test_extract_filename4-1.csv']
+
+        given_result = self.process_tracker.find_ready_extracts_by_location('Test Location')
+
+        self.assertNotEqual(expected_result, given_result)
+
     def test_find_ready_extracts_by_process(self):
         """
         Testing that for the given process name, find the extracts, provided they are in 'ready' state.
         :return:
         """
         extract = ExtractTracker(process_run=self.process_tracker
-                       , filename='test_extract_filename5.csv'
+                       , filename='test_extract_filename5-1.csv'
+                       , location_name='Test Location'
+                       , location_path='/home/test/extract_dir')
+
+        extract2 = ExtractTracker(process_run=self.process_tracker
+                       , filename='test_extract_filename5-2.csv'
                        , location_name='Test Location'
                        , location_path='/home/test/extract_dir')
 
@@ -141,12 +232,49 @@ def test_find_ready_extracts_by_process(self):
         session = Session.object_session(extract.extract)
         session.commit()
 
-        expected_result = ['/home/test/extract_dir/test_extract_filename5.csv']
+        extract2.extract.extract_status_id = extract2.extract_status_ready
+        session = Session.object_session(extract2.extract)
+        session.commit()
+
+        expected_result = ['/home/test/extract_dir/test_extract_filename5-1.csv'
+                           , '/home/test/extract_dir/test_extract_filename5-2.csv']
 
         given_result = self.process_tracker.find_ready_extracts_by_process('Testing Process Tracking Initialization')
 
         self.assertEqual(expected_result, given_result)
 
+    def test_find_ready_extracts_by_process_not_descending(self):
+        """
+        Testing that for the given process name, find the extracts, provided they are in 'ready' state.  Verifying that
+        records are NOT returned in Descending order.
+        :return:
+        """
+        extract = ExtractTracker(process_run=self.process_tracker
+                                 , filename='test_extract_filename5-1.csv'
+                                 , location_name='Test Location'
+                                 , location_path='/home/test/extract_dir')
+
+        extract2 = ExtractTracker(process_run=self.process_tracker
+                                  , filename='test_extract_filename5-2.csv'
+                                  , location_name='Test Location'
+                                  , location_path='/home/test/extract_dir')
+
+        # Need to manually change the status, because this would normally be done while the process was processing data
+        extract.extract.extract_status_id = extract.extract_status_ready
+        session = Session.object_session(extract.extract)
+        session.commit()
+
+        extract2.extract.extract_status_id = extract2.extract_status_ready
+        session = Session.object_session(extract2.extract)
+        session.commit()
+
+        expected_result = ['/home/test/extract_dir/test_extract_filename5-2.csv'
+            , '/home/test/extract_dir/test_extract_filename5-1.csv']
+
+        given_result = self.process_tracker.find_ready_extracts_by_process('Testing Process Tracking Initialization')
+
+        self.assertNotEqual(expected_result, given_result)
+
     def test_initializing_process_tracking(self):
         """
         Testing that when ProcessTracking is initialized, the necessary objects are created.