Update all dependencies and bump to 2.3.6

Author: rogerdahl

README.md

@@ -110,16 +110,16 @@ When unit tests are being run as part of CI or as a normal guard against regress
 
 The normal procedure for writing a sample based unit test is to just write the test as if the sample already exists, then running the test with `--sample-ask` and viewing and approving the resulting sample, which is then automatically written to a file. The sample file name is displayed, making it easy to find the file in order to add it to tracking so that it can be committed along with the test module.
 
-When working on large changes that cause many samples to become outdated, reviewing and approving samples can be deferred until the new code approaches stability. This is done by running the tests with `--sample-write`, which automatically writes or updates samples to match the current results. Then, running the tests with `--sample-review` before committing.  
+When working on large changes that cause many samples to become outdated, reviewing and approving samples can be deferred until the new code approaches stability. This is done by running the tests with `--sample-write`, which automatically writes or updates samples to match the current results. Then, view and approve the tests with `--sample-review` before committing.  
 
 Typically, it is not desirable to track generated files in Git. However, although the sample files are generated, they are an integral part of the units tests, and should be tracked just like the unit tests themselves.
 
-Also implemented is a simple process for cleaning out unused sample files. Sample files are often orphaned when their corresponding tests are removed or refactored. The default storage location for sample files is a directory called `test_docs`. To clean out unused files, move all the files from `test_docs` to `test_docs_tidy`, and run the tests. Any files that are accessed by the tests will automatically be moved back to `test_docs`, and any files remaining in `test_docs_tidy` after a complete test run can be untracked and deleted.
+Also implemented is a simple process for cleaning out unused sample files. Sample files are often orphaned when their corresponding tests are removed or refactored. The process is activated with the `--sample-tidy` switch. When active, the test session starts by moving all sample files from their default directory, `test_docs`, to `test_docs_tidy`. As the sample files are accessed by tests, they are automatically moved back to `test_docs`, and any files remaining in `test_docs_tidy` after a complete test run can be untracked and deleted.
 
 When staging `test_docs`, stage the directory, so that new files are included, and deleted files get deleted on the server:
 
     $ git add test_utilities/src/d1_test/test_docs
-    $ git commit -m 'Update sample docs'
+    $ git commit -m 'Update samples'
 
 #### DataONE Client to Django test adapter
 
@@ -226,7 +226,9 @@ Edit `~/.pypirc`:
 
 ### Creating a new release
 
-The stack should pass all the tests with the most recent available versions of all the dependencies.
+#### Updating dependencies
+
+Update all packages managed by pip:
 
     $ cd d1_python
     $ sudo ./dev_tools/src/d1_dev/pip-update-all.py
@@ -244,10 +246,12 @@ Check that there are no package version conflicts:
 
 Some sample files contain the version tag and must be updated when the version changes.  
 
-    $ pytest --sample-ask
+    $ pytest -xvs --sample-ask
 
 Commit and push the changes, and check the build on Travis.
 
+#### Building the release packages
+
 After successful build, clone a fresh copy, which will be used for building the release packages:
 
     $ cd ~

client_cli/src/setup.py

@@ -28,7 +28,7 @@
 def main():
   setuptools.setup(
     name='dataone.cli',
-    version='2.3.5',
+    version='2.3.6',
     description='Command-Line Interface (CLI) for DataONE',
     author='DataONE Project',
     author_email='developers@dataone.org',
@@ -37,10 +37,10 @@ def main():
     packages=setuptools.find_packages(),
     include_package_data=True,
     install_requires=[
-      'dataone.common == 2.3.5',
-      'dataone.libclient == 2.3.5',
+      'dataone.common == 2.3.6',
+      'dataone.libclient == 2.3.6',
       #
-      'requests == 2.18.3',
+      'requests == 2.18.4',
     ],
     entry_points={'console_scripts': [
       'dataone = d1_cli.dataone:main',

client_onedrive/src/setup.py

@@ -32,7 +32,7 @@
 def main():
   setuptools.setup(
     name='dataone.onedrive',
-    version='2.3.5',
+    version='2.3.6',
     description='Filesystem access to the DataONE Workspace',
     author='DataONE Project',
     author_email='developers@dataone.org',
@@ -41,14 +41,14 @@ def main():
     packages=setuptools.find_packages(),
     include_package_data=True,
     install_requires=[
-      'dataone.common == 2.3.5',
-      'dataone.libclient == 2.3.5',
+      'dataone.common == 2.3.6',
+      'dataone.libclient == 2.3.6',
       #
       'fusepy == 2.0.4',
       'pyxb == 1.2.5',
       'pyzotero == 1.2.13',
       'rdflib == 4.2.2',
-      'requests == 2.18.3',
+      'requests == 2.18.4',
     ],
     setup_requires=[
       'setuptools_git >= 1.1',

dev_tools/src/setup.py

@@ -29,7 +29,7 @@
 def main():
   setuptools.setup(
     name='dataone.dev',
-    version='2.3.5',
+    version='2.3.6',
     description='DataONE developer tools',
     author='DataONE Project',
     author_email='developers@dataone.org',
@@ -50,7 +50,7 @@ def main():
       # asn1crypto and requests. Without this, if asn1crypto is installed first,
       # it will install idna 2.6, and requests will fails due to the fact that
       # Python doesn't have a real package manager.
-      'idna == 2.5',
+      'idna == 2.6',
       #
       'baron == 0.6.6',
       'pip == 9.0.1',

gmn/src/d1_gmn/app/views/slice.py

@@ -22,7 +22,6 @@
 
 import copy
 import hashlib
-import logging
 
 import d1_common.const
 import d1_common.types
@@ -33,21 +32,23 @@
 import django.core.cache
 import django.db.models
 
+# import logging
+
 
 def add_slice_filter(request, query, total_int):
   url_dict = d1_common.url.parseUrl(request.get_full_path())
   query_dict = url_dict['query']
-  logging.error(query_dict)
+  # logging.debug(query_dict)
   start_int, count_int, total_int = (
     _convert_and_sanity_check_slice_params(query_dict, total_int)
   )
-  logging.error(query_dict)
-  logging.error(url_dict)
+  # logging.debug(query_dict)
+  # logging.debug(url_dict)
   authn_subj_set = _get_authenticated_subjects(request)
-  logging.debug(
-    'Adding slice filter. start={} count={} total={} subj={}'
-    .format(start_int, count_int, total_int, ','.join(list(authn_subj_set)))
-  )
+  # logging.debug(
+  #   'Adding slice filter. start={} count={} total={} subj={}'
+  #   .format(start_int, count_int, total_int, ','.join(list(authn_subj_set)))
+  # )
   last_tup = _cache_get_last_in_slice(url_dict, authn_subj_set)
   if last_tup:
     query = _add_fast_slice_filter(query, last_tup, count_int)
@@ -70,7 +71,7 @@ def cache_add_last_in_slice(request, query, total_int, sort_field_list):
   else:
     last_tup = None
   django.core.cache.cache.set(key_str, last_tup)
-  logging.debug('Cache set. key="{}" last={}'.format(key_str, last_tup))
+  # logging.debug('Cache set. key="{}" last={}'.format(key_str, last_tup))
 
 
 #
@@ -128,9 +129,9 @@ def _get_authenticated_subjects(request):
 
 
 def _add_fast_slice_filter(query, last_tup, count_int):
-  logging.debug(
-    'Adding fast slice filter. last={} count={}'.format(last_tup, count_int)
-  )
+  # logging.debug(
+  #   'Adding fast slice filter. last={} count={}'.format(last_tup, count_int)
+  # )
   last_timestamp, last_id = last_tup
   return query.filter(
     django.db.models.Q(timestamp__lt=last_timestamp) |
@@ -145,10 +146,10 @@ def _add_fallback_slice_filter(query, url_dict):
   to run very slowly on large result sets.
   """
   start_int, count_int, total_int = _get_slice_params(url_dict['query'])
-  logging.debug(
-    'Adding fallback slice filter. start={} count={}'.
-    format(start_int, count_int)
-  )
+  # logging.debug(
+  #   'Adding fallback slice filter. start={} count={}'.
+  #   format(start_int, count_int)
+  # )
   if not start_int and not count_int:
     return query.none()
   else:
@@ -163,7 +164,7 @@ def _cache_get_last_in_slice(url_dict, authn_subj_set):
     last_tup = django.core.cache.cache.get(key_str)
   except KeyError:
     last_tup = None
-  logging.debug('Cache get. key="{}" -> last_tup={}'.format(key_str, last_tup))
+  # logging.debug('Cache get. key="{}" -> last_tup={}'.format(key_str, last_tup))
   return last_tup
 
 
@@ -191,7 +192,7 @@ def _gen_cache_key_for_slice(url_dict, authn_subj_set, result_record_count=0):
   hash of the JSON is used for better distribution in a hash map and to avoid
   the 256 bytes limit on keys in some caches.
   """
-  logging.debug('Gen key. result_record_count={}'.format(result_record_count))
+  # logging.debug('Gen key. result_record_count={}'.format(result_record_count))
   key_url_dict = copy.deepcopy(url_dict)
   key_url_dict['query']['start'] += result_record_count
   del key_url_dict['query']['count']

gmn/src/d1_gmn/tests/gmn_test_case.py

@@ -72,33 +72,37 @@ class GMNTestCase(
     d1_test.d1_test_case.D1TestCase,
 ):
   def setup_class(self):
+    """Run for each test class that derives from GMNTestCase"""
     if ENABLE_SQL_PROFILING:
       connection.queries = []
 
   def teardown_class(self):
+    """Run for each test class that derives from GMNTestCase"""
     GMNTestCase.capture_exception()
     if ENABLE_SQL_PROFILING:
-      logging.debug('SQL queries by all methods:')
-      map(logging.debug, connection.queries)
+      logging.info('SQL queries by all methods:')
+      map(logging.info, connection.queries)
 
   def setup_method(self, method):
+    """Run for each test method that derivces from GMNTestCase"""
+    # logging.error('GMNTestCase.setup_method()')
     d1_test.mock_api.django_client.add_callback(
       d1_test.d1_test_case.MOCK_BASE_URL
     )
-    d1_test.mock_api.get.add_callback(d1_test.d1_test_case.MOCK_BASE_URL)
+    # d1_test.mock_api.get.add_callback(d1_test.d1_test_case.MOCK_BASE_URL)
     self.client_v1 = d1_client.mnclient_1_1.MemberNodeClient_1_1(
       d1_test.d1_test_case.MOCK_BASE_URL
     )
     self.client_v2 = d1_client.mnclient_2_0.MemberNodeClient_2_0(
       d1_test.d1_test_case.MOCK_BASE_URL
     )
-    self.test_client = d1_gmn.tests.gmn_test_client.GMNTestClient(
-      d1_test.d1_test_case.MOCK_BASE_URL
-    )
+    # self.test_client = d1_gmn.tests.gmn_test_client.GMNTestClient(
+    #   d1_test.d1_test_case.MOCK_BASE_URL
+    # )
     self.v1 = d1_common.types.dataoneTypes_v1_1
     self.v2 = d1_common.types.dataoneTypes_v2_0
-    # Remove limit on max diff to show. This can cause debug output to
-    # explode...
+    # # Remove limit on max diff to show. This can cause debug output to
+    # # explode...
     self.maxDiff = None
 
   @property

gmn/src/d1_gmn/tests/test_archive_revision.py

@@ -32,28 +32,31 @@ class TestArchiveRevision(d1_gmn.tests.gmn_test_case.GMNTestCase):
   @responses.activate
   def test_1000(self, cn_mn_client_v1_v2):
     """archive(): Archived flag correctly set and represented"""
-    with d1_gmn.tests.gmn_mock.disable_auth():
 
-      def assert_archived(client, pid):
-        unarchived_sysmeta_pyxb = client.getSystemMetadata(pid)
-        assert not unarchived_sysmeta_pyxb.archived
-        pid_archived = client.archive(pid)
-        assert pid == pid_archived.value()
-        archived_sysmeta_pyxb = client.getSystemMetadata(pid)
-        assert archived_sysmeta_pyxb.archived
-        self.assert_eq_sysmeta_sid(
-          unarchived_sysmeta_pyxb,
-          archived_sysmeta_pyxb,
-        )
+    def assert_archived(client, pid):
+      unarchived_sysmeta_pyxb = client.getSystemMetadata(pid)
+      assert not unarchived_sysmeta_pyxb.archived
+      pid_archived = client.archive(pid)
+      assert pid == pid_archived.value()
+      archived_sysmeta_pyxb = client.getSystemMetadata(pid)
+      assert archived_sysmeta_pyxb.archived
+      self.assert_eq_sysmeta_sid(
+        unarchived_sysmeta_pyxb,
+        archived_sysmeta_pyxb,
+      )
+
+    # archive() is supported on both CNs and MNs. Since this test creates a
+    # revision chain, and create() is only supported on MNs, we use the MN
+    # client to create the objects when testing both CN and MN archive().
 
-      # archive() is supported on both CNs and MNs. Since this test creates a
-      # revision chain, and create() is only supported on MNs, we use the MN
-      # client to create the objects when testing both CN and MN archive().
-      # with d1_gmn.tests.gmn_mock.disable_auth():
+    # mock_get_system_metadata.add_callback(d1_test.d1_test_case.MOCK_BASE_URL)
+
+    with d1_gmn.tests.gmn_mock.disable_auth():
 
       base_sid, pid_chain_list = self.create_revision_chain(
         self.client_v2, chain_len=5, sid=True, disable_auth=False
       )
+
       self.assert_valid_chain(self.client_v2, pid_chain_list, base_sid)
 
       # Archive head

gmn/src/d1_gmn/tests/test_create_and_get_standalone.py

@@ -106,7 +106,6 @@ def test_1050(self, mn_client_v1_v2):
     """create() / get(): Object with no explicit permissions can be retrieved
     by a trusted subject
     """
-
     pid, sid, sciobj_str, sysmeta_pyxb = self.create_obj(mn_client_v1_v2)
     self.get_obj(
       mn_client_v1_v2,

gmn/src/d1_gmn/tests/test_slice.py

@@ -27,6 +27,8 @@
 import datetime
 import random
 
+# import logging
+# import pytest
 import responses
 
 import d1_gmn.app.models
@@ -36,15 +38,20 @@
 
 import d1_test.sample
 
-# import logging
-
 
 @d1_test.d1_test_case.reproducible_random_decorator('TestSlice')
-# @freezegun.freeze_time('1978-05-27')
+# @pytest.mark.skip('SLOOOOOW')
 class TestSlice(d1_gmn.tests.gmn_test_case.GMNTestCase):
-  def norm_entry_id(self, log):
-    for log_entry_pyxb in log.logEntry:
-      log_entry_pyxb.entryId = '1'
+  # def setup_class(self):
+  #   with d1_gmn.tests.gmn_mock.disable_auth():
+  #     self._total_int = mn_client_v1_v2.getLogRecords(
+  #       start=0, count=0, fromDate=fromDate
+  #     ).total
+  #     assert self._total_int >= 2000, 'Need >= 2000 logEntry records for testing'
+  #     single_slice_log = mn_client_v1_v2.getLogRecords(
+  #       start=0, count=self._total_int, fromDate=fromDate
+  #     )
+  #     assert len(single_slice_log.logEntry) == self._total_int
 
   def _test(self, mn_client_v1_v2, fromDate=None, vary_size=False):
     count_int = 789

gmn/src/d1_gmn/version.py

@@ -1 +1 @@
-__version__ = '2.3.5'
+__version__ = '2.3.6'