Merge branch 'main' into ccm-UID2-3497-implement-identity-buckets

Author: caroline-ttd

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "uid2_client"
-version = "3.0.0a1"
+version = "2.4.6"
 authors = [
     { name = "UID2 team", email = "unifiedid-admin@thetradedesk.com" }
 ]
@@ -18,9 +18,9 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.6"
 dependencies = [
-    "requests",
+    "setuptools",
     "pycryptodome",
     "bitarray"
 ]

tests/test_identity_map_client.py

@@ -1,9 +1,8 @@
-import datetime
+import datetime as dt
 import os
 import unittest
-from datetime import datetime
 
-import requests
+from urllib.error import URLError, HTTPError
 
 from uid2_client import IdentityMapClient, IdentityMapInput, normalize_and_hash_email, normalize_and_hash_phone, \
     get_datetime_utc_iso_format
@@ -137,27 +136,26 @@ def test_identity_map_hashed_phones(self):
     def test_identity_map_client_bad_url(self):
         identity_map_input = IdentityMapInput.from_emails(
             ["hopefully-not-opted-out@example.com", "somethingelse@example.com", "optout@example.com"])
-        client = IdentityMapClient("https://operator-bad-url.uidapi.com", os.getenv("UID2_API_KEY"),
-                                   os.getenv("UID2_SECRET_KEY"))
-        self.assertRaises(requests.exceptions.ConnectionError, client.generate_identity_map, identity_map_input)
-        self.assertRaises(requests.exceptions.ConnectionError, client.get_identity_buckets, datetime.datetime.now())
+        client = IdentityMapClient("https://operator-bad-url.uidapi.com", os.getenv("UID2_API_KEY"), os.getenv("UID2_SECRET_KEY"))
+        self.assertRaises(URLError, client.generate_identity_map, identity_map_input)
+        self.assertRaises(URLError, client.get_identity_buckets, dt.datetime.now())
 
     def test_identity_map_client_bad_api_key(self):
         identity_map_input = IdentityMapInput.from_emails(
             ["hopefully-not-opted-out@example.com", "somethingelse@example.com", "optout@example.com"])
         client = IdentityMapClient(os.getenv("UID2_BASE_URL"), "bad-api-key", os.getenv("UID2_SECRET_KEY"))
-        self.assertRaises(requests.exceptions.HTTPError, client.generate_identity_map, identity_map_input)
-        self.assertRaises(requests.exceptions.HTTPError, client.get_identity_buckets, datetime.datetime.now())
+        self.assertRaises(HTTPError, client.generate_identity_map,identity_map_input)
+        self.assertRaises(HTTPError, client.get_identity_buckets, dt.datetime.now())
 
     def test_identity_map_client_bad_secret(self):
         identity_map_input = IdentityMapInput.from_emails(
             ["hopefully-not-opted-out@example.com", "somethingelse@example.com", "optout@example.com"])
-        client = IdentityMapClient(os.getenv("UID2_BASE_URL"), os.getenv("UID2_API_KEY"),
-                                   "wJ0hP19QU4hmpB64Y3fV2dAed8t/mupw3sjN5jNRFzg=")
-        self.assertRaises(requests.exceptions.HTTPError, client.generate_identity_map,
+
+        client = IdentityMapClient(os.getenv("UID2_BASE_URL"), os.getenv("UID2_API_KEY"), "wJ0hP19QU4hmpB64Y3fV2dAed8t/mupw3sjN5jNRFzg=")
+        self.assertRaises(HTTPError, client.generate_identity_map,
                           identity_map_input)
-        self.assertRaises(requests.exceptions.HTTPError, client.get_identity_buckets,
-                          datetime.datetime.now())
+        self.assertRaises(HTTPError, client.get_identity_buckets,
+                          dt.datetime.now())
 
     def assert_mapped(self, response, dii):
         mapped_identity = response.mapped_identities.get(dii)
@@ -176,12 +174,12 @@ def assert_unmapped(self, response, reason, dii):
         self.assertIsNone(mapped_identity)
 
     def test_identity_buckets(self):
-        response = self.identity_map_client.get_identity_buckets(datetime.datetime.now() - datetime.timedelta(days=90))
+        response = self.identity_map_client.get_identity_buckets(dt.datetime.now() - dt.timedelta(days=90))
         self.assertTrue(len(response.buckets) > 0)
         self.assertTrue(response.is_success)
 
     def test_identity_buckets_empty_response(self):
-        response = self.identity_map_client.get_identity_buckets(datetime.datetime.now() + datetime.timedelta(days=1))
+        response = self.identity_map_client.get_identity_buckets(dt.datetime.now() + dt.timedelta(days=1))
         self.assertTrue(len(response.buckets) == 0)
         self.assertTrue(response.is_success)
 
@@ -204,7 +202,7 @@ def test_get_datetime_utc_iso_format_timestamp(self):
                       "2024-07-02T06:30:15.123456-08:00", "2024-07-02T23:30:15.123456+09:00",
                       "2024-07-03T00:30:15.123456+10:00", "2024-07-02T20:00:15.123456+05:30"]
         for timestamp_str in test_cases:
-            timestamp = datetime.fromisoformat(timestamp_str)
+            timestamp = dt.datetime.fromisoformat(timestamp_str)
             iso_format_timestamp = get_datetime_utc_iso_format(timestamp)
             self.assertEqual(expected_timestamp, iso_format_timestamp)
 

tests/test_publisher_client.py

@@ -1,11 +1,11 @@
 import os
 import unittest
 
-import requests
-
 from uid2_client import Uid2PublisherClient
 from uid2_client import TokenGenerateInput
+from uid2_client import TokenGenerateResponse
 from uid2_client.identity_tokens import IdentityTokens
+from urllib.request import HTTPError
 
 
 class PublisherEuidIntegrationTests(unittest.TestCase):
@@ -175,7 +175,7 @@ def test_integration_bad_requests(self):
 
         expired_respose = "{\"advertising_token\":\"AgAAAAN6QZRCFTau+sfOlMMUY2ftElFMq2TCrcu1EAaD9WmEfoT2BWm2ZKz1tumbT00tWLffRDQ/9POXfA0O/Ljszn7FLtG5EzTBM3HYs4f5irkqeEvu38DhVCxUEpI+gZZZkynRap1oYx6AmC/ip3rk+7pmqa3r3saDs1mPRSSTm+Nh6A==\",\"user_token\":\"AgAAAAL6aleYI4BubI5ZXMBshqmMEfCkbCJF4fLeg1sdI0BTLzj9sXsSISjkG0lMC743diC2NVy3ElkbO1lLysd+Lm6alkqevPrcuWDisQ1939YdoH6LqpwBH3FNSE4/xa3Q+94=\",\"refresh_token\":\"AAAAAARomrP3NjjH+8mt5djfTHbmRZXjOMnAN8WpjJoe30AhUCvYksO/xoDSj77GzWv4M99DhnPl2cVco8CZFTcE10nauXI4Barr890ILnH0IIacOei5Zjwh6DycFkoXkAAuHY1zjmxb7niGLfSP2RctWkZdRVGWQv/UW/grw6+paU9bnKEWPzVvLwwdW2NgjDKu+szE6A+b5hkY+I3voKoaz8/kLDmX8ddJGLy/YOh/LIveBspSAvEg+v89OuUCwAqm8L3Rt8PxDzDnt0U4Na+AUawvvfsIhmsn/zMpRRks6GHhIAB/EQUHID8TedU8Hv1WFRsiraG9Dfn1Kc5/uYnDJhEagWc+7RgTGT+U5GqI6+afrAl5091eBLbmvXnXn9ts\",\"identity_expires\":1668059799628,\"refresh_expires\":1668142599628,\"refresh_from\":1668056202628,\"refresh_response_key\":\"P941vVeuyjaDRVnFQ8DPd0AZnW4bPeiJPXER2K9QXcU=\"}"
         current_identity = IdentityTokens.from_json_string(expired_respose)
-        with self.assertRaises(requests.exceptions.HTTPError):
+        with self.assertRaises(HTTPError):
             self.publisher_client.refresh_token(current_identity)
 
         with self.assertRaises(TypeError):
@@ -185,15 +185,15 @@ def test_integration_bad_requests(self):
             self.publisher_client.refresh_token(None)
 
         bad_url_client = Uid2PublisherClient("https://www.something.com", self.UID2_API_KEY, self.UID2_SECRET_KEY)
-        with self.assertRaises(requests.exceptions.HTTPError):
+        with self.assertRaises(HTTPError):
             bad_url_client.generate_token(TokenGenerateInput.from_email("test@example.com"))
 
         bad_secret_client = Uid2PublisherClient(self.UID2_BASE_URL, self.UID2_API_KEY, "badSecretKeypB64Y3fV2dAed8t/mupw3sjN5jNRFzg=")
-        with self.assertRaises(requests.exceptions.HTTPError):
+        with self.assertRaises(HTTPError):
             bad_secret_client.generate_token(TokenGenerateInput.from_email("test@example.com"))
 
         bad_api_client = Uid2PublisherClient(self.UID2_BASE_URL, "not-real-key", self.UID2_SECRET_KEY)
-        with self.assertRaises(requests.exceptions.HTTPError):
+        with self.assertRaises(HTTPError):
             bad_secret_client.generate_token(TokenGenerateInput.from_email("test@example.com"))
 
 

tests/test_refresh_keys_util.py

@@ -1,14 +1,20 @@
 import json
 import unittest
-
-import responses
+from unittest.mock import patch
 
 from uid2_client import refresh_keys_util
 from test_utils import *
 from uid2_client.encryption import _encrypt_gcm, _decrypt_gcm
 
 
 class TestRefreshKeysUtil(unittest.TestCase):
+    class MockPostResponse:
+        def __init__(self, return_value):
+            self.return_value = return_value
+
+        def read(self):
+            return base64.b64encode(self.return_value)
+
     def _make_post_response(self, request_data, response_payload):
         d = base64.b64decode(request_data)[1:]
         d = _decrypt_gcm(d, client_secret_bytes)
@@ -19,11 +25,11 @@ def _make_post_response(self, request_data, response_payload):
         payload += response_payload
         envelope = _encrypt_gcm(payload, None, client_secret_bytes)
 
-        return 200, {}, base64.b64encode(envelope)
+        return self.MockPostResponse(envelope)
 
-    def _get_post_refresh_keys_response(self, request):
+    def _get_post_refresh_keys_response(self, base_url, path, headers, data):
         response_payload = key_set_to_json_for_sharing([master_key, site_key]).encode()
-        return self._make_post_response(request.body, response_payload)
+        return self._make_post_response(data, response_payload)
 
     def _validate_master_and_site_key(self, keys):
         self.assertEqual(len(keys.values()), 2)
@@ -49,29 +55,23 @@ def _validate_master_and_site_key(self, keys):
         self.assertEqual(master_secret, master.secret)
         self.assertEqual(1, master.keyset_id)
 
-    @responses.activate
-    def test_refresh_sharing_keys(self):
-        responses.add_callback(
-            responses.POST,
-            "https://base_url/v2/key/sharing",
-            callback=self._get_post_refresh_keys_response,
-        )
-
-        refresh_response = refresh_keys_util.refresh_sharing_keys("https://base_url", "auth_key", base64.b64decode(client_secret))
+    @patch('uid2_client.refresh_keys_util.post')
+    def test_refresh_sharing_keys(self, mock_post):
+        mock_post.side_effect = self._get_post_refresh_keys_response
+        refresh_response = refresh_keys_util.refresh_sharing_keys("base_url", "auth_key", base64.b64decode(client_secret))
         self.assertTrue(refresh_response.success)
         self._validate_master_and_site_key(refresh_response.keys)
+        mock_post.assert_called_once()
+        self.assertEqual(mock_post.call_args[0], ('base_url', '/v2/key/sharing'))
 
-    @responses.activate
-    def test_refresh_bidstream_keys(self):
-        responses.add_callback(
-            responses.POST,
-            "https://base_url/v2/key/bidstream",
-            callback=self._get_post_refresh_keys_response,
-        )
-
-        refresh_response = refresh_keys_util.refresh_bidstream_keys("https://base_url", "auth_key", base64.b64decode(client_secret))
+    @patch('uid2_client.refresh_keys_util.post')
+    def test_refresh_bidstream_keys(self, mock_post):
+        mock_post.side_effect = self._get_post_refresh_keys_response
+        refresh_response = refresh_keys_util.refresh_bidstream_keys("base_url", "auth_key", base64.b64decode(client_secret))
         self.assertTrue(refresh_response.success)
         self._validate_master_and_site_key(refresh_response.keys)
+        mock_post.assert_called_once()
+        self.assertEqual(mock_post.call_args[0], ('base_url', '/v2/key/bidstream'))
 
     def test_parse_keys_json_identity(self):
         response_body_str = key_set_to_json_for_sharing([master_key, site_key])

uid2_client/identity_map_client.py

@@ -38,14 +38,12 @@ def generate_identity_map(self, identity_map_input):
         req, nonce = make_v2_request(self._client_secret, dt.datetime.now(tz=timezone.utc),
                                      identity_map_input.get_identity_map_input_as_json_string().encode())
         resp = post(self._base_url, '/v2/identity/map', headers=auth_headers(self._api_key), data=req)
-        resp.raise_for_status()
-        resp_body = parse_v2_response(self._client_secret, resp.text, nonce)
+        resp_body = parse_v2_response(self._client_secret, resp.read(), nonce)
         return IdentityMapResponse(resp_body, identity_map_input)
 
     def get_identity_buckets(self, since_timestamp):
         req, nonce = make_v2_request(self._client_secret, dt.datetime.now(tz=timezone.utc),
                                      json.dumps({"since_timestamp": get_datetime_utc_iso_format(since_timestamp)}).encode())
         resp = post(self._base_url, '/v2/identity/buckets', headers=auth_headers(self._api_key), data=req)
-        resp.raise_for_status()
-        resp_body = parse_v2_response(self._client_secret, resp.text, nonce)
+        resp_body = parse_v2_response(self._client_secret, resp.read(), nonce)
         return IdentityBucketsResponse(resp_body)

uid2_client/publisher_client.py

@@ -50,14 +50,12 @@ def generate_token(self, token_generate_input):
         req, nonce = make_v2_request(self._secret_key, dt.datetime.now(tz=timezone.utc),
                                      token_generate_input.get_as_json_string().encode())
         resp = post(self._base_url, '/v2/token/generate', headers=auth_headers(self._auth_key), data=req)
-        resp.raise_for_status()
-        resp_body = parse_v2_response(self._secret_key, resp.text, nonce)
+        resp_body = parse_v2_response(self._secret_key, resp.read(), nonce)
         return TokenGenerateResponse(resp_body)
 
     def refresh_token(self, current_identity):
         resp = post(self._base_url, '/v2/token/refresh', headers=auth_headers(self._auth_key),
                     data=current_identity.get_refresh_token().encode())
-        resp.raise_for_status()
-        resp_bytes = base64_to_byte_array(resp.text)
+        resp_bytes = base64_to_byte_array(resp.read())
         decrypted = _decrypt_gcm(resp_bytes, base64_to_byte_array(current_identity.get_refresh_response_key()))
         return TokenRefreshResponse(decrypted.decode(), dt.datetime.now(tz=timezone.utc))

uid2_client/refresh_keys_util.py

@@ -38,8 +38,7 @@ def _fetch_keys(base_url, path, auth_key, secret_key):
     try:
         req, nonce = make_v2_request(secret_key, dt.datetime.now(tz=timezone.utc))
         resp = post(base_url, path, headers=auth_headers(auth_key), data=req)
-        resp.raise_for_status()
-        resp_body = json.loads(parse_v2_response(secret_key, resp.text, nonce)).get('body')
+        resp_body = json.loads(parse_v2_response(secret_key, resp.read(), nonce)).get('body')
         keys = _parse_keys_json(resp_body)
         return RefreshResponse.make_success(keys)
     except Exception as exc:

uid2_client/request_response_util.py

@@ -1,7 +1,8 @@
 import base64
-from importlib.metadata import version
 import os
-import requests
+from urllib import request
+
+import pkg_resources
 
 from uid2_client.encryption import _encrypt_gcm, _decrypt_gcm
 
@@ -12,12 +13,12 @@ def _make_url(base_url, path):
 
 def auth_headers(auth_key):
     try:
-        client_version = version("uid2_client")
+        version = pkg_resources.get_distribution("uid2_client").version
     except Exception:
-        client_version = "non-packaged-mode"
+        version = "non-packaged-mode"
 
     return {'Authorization': 'Bearer ' + auth_key,
-            "X-UID2-Client-Version": "uid2-client-python-" + client_version}
+            "X-UID2-Client-Version": "uid2-client-python-" + version}
 
 
 def make_v2_request(secret_key, now, data=None):
@@ -41,4 +42,5 @@ def parse_v2_response(secret_key, encrypted, nonce):
 
 
 def post(base_url, path, headers, data):
-    return requests.post(_make_url(base_url, path), data=data, headers=headers)
+    req = request.Request(_make_url(base_url, path), headers=headers, method='POST', data=data)
+    return request.urlopen(req)

uid2_client/uid2_token_generator.py

@@ -70,6 +70,7 @@ class UID2TokenGenerator:
 
     @staticmethod
     def generate_uid2_token_v2(id_str, master_key, site_id, site_key, params=None, version=2):
+        """This function is only used by tests."""
         if params is None:
             params = Params()
 
@@ -80,11 +81,11 @@ def generate_uid2_token_v2(id_str, master_key, site_id, site_key, params=None, v
         # old privacy_bits
         identity += int.to_bytes(0, 4, 'big')
         identity += int.to_bytes(int(params.identity_established.timestamp()) * 1000, 8, 'big')
-        identity_iv = bytes([10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+        identity_iv = os.urandom(16)
         expiry = params.token_expiry
         master_payload = int.to_bytes(int(expiry.timestamp()) * 1000, 8, 'big')
         master_payload += _encrypt_data_v1(identity, key=site_key, iv=identity_iv)
-        master_iv = bytes([21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36])
+        master_iv = os.urandom(16)
 
         token = int.to_bytes(version, 1, 'big')
         token += _encrypt_data_v1(master_payload, key=master_key, iv=master_iv)