Skip to content

Commit 607034a

Browse files
jaeoptjaeopt
committed
[AI-FSSDK] [FSSDK-12262] Exclude CMAB from UserProfileService
1 parent 88b0644 commit 607034a

File tree

2 files changed

+216
-2
lines changed

2 files changed

+216
-2
lines changed

‎optimizely/decision_service.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,14 @@ def get_variation(
457457
}
458458

459459
# Check to see if user has a decision available for the given experiment
460-
if user_profile_tracker is not None and not ignore_user_profile:
460+
# CMAB experiments are excluded from UPS because UPS maintains decisions
461+
# across the experiment lifetime without considering TTL or user attributes,
462+
# which contradicts CMAB's dynamic nature.
463+
if experiment.cmab:
464+
message = f'Skipping user profile service for CMAB experiment "{experiment.key}".'
465+
self.logger.info(message)
466+
decide_reasons.append(message)
467+
elif user_profile_tracker is not None and not ignore_user_profile:
461468
variation = self.get_stored_variation(project_config, experiment, user_profile_tracker.get_user_profile())
462469
if variation:
463470
message = f'Returning previously activated variation ID "{variation}" of experiment ' \
@@ -529,7 +536,8 @@ def get_variation(
529536
self.logger.info(message)
530537
decide_reasons.append(message)
531538
# Store this new decision and return the variation for the user
532-
if user_profile_tracker is not None and not ignore_user_profile:
539+
# CMAB experiments are excluded from UPS to preserve dynamic decision-making
540+
if user_profile_tracker is not None and not ignore_user_profile and not experiment.cmab:
533541
try:
534542
user_profile_tracker.update_user_profile(experiment, variation)
535543
except:

‎tests/test_decision_service.py‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1890,3 +1890,209 @@ def test_get_variation_for_feature_returns_rollout_in_experiment_bucket_range_25
18901890
mock_config_logging.debug.assert_called_with(
18911891
'Assigned bucket 4000 to user with bucketing ID "test_user".')
18921892
mock_generate_bucket_value.assert_called_with("test_user211147")
1893+
1894+
def test_get_variation_cmab_experiment_skips_ups_lookup(self):
1895+
"""Test that CMAB experiments skip user profile service lookup."""
1896+
1897+
user = optimizely_user_context.OptimizelyUserContext(
1898+
optimizely_client=None,
1899+
logger=None,
1900+
user_id="test_user",
1901+
user_attributes={}
1902+
)
1903+
1904+
cmab_experiment = entities.Experiment(
1905+
'111150',
1906+
'cmab_experiment',
1907+
'Running',
1908+
'111150',
1909+
[],
1910+
{},
1911+
[
1912+
entities.Variation('111151', 'variation_1'),
1913+
entities.Variation('111152', 'variation_2')
1914+
],
1915+
[
1916+
{'entityId': '111151', 'endOfRange': 5000},
1917+
{'entityId': '111152', 'endOfRange': 10000}
1918+
],
1919+
cmab={'trafficAllocation': 5000}
1920+
)
1921+
1922+
# Create a user profile tracker with a stored variation for this experiment
1923+
ups = user_profile.UserProfileService()
1924+
tracker = user_profile.UserProfileTracker("test_user", ups, self.decision_service.logger)
1925+
tracker.user_profile = user_profile.UserProfile(
1926+
"test_user",
1927+
{"111150": {"variation_id": "111151"}}
1928+
)
1929+
1930+
with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \
1931+
mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \
1932+
mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id',
1933+
return_value=['$', []]), \
1934+
mock.patch.object(self.decision_service, 'cmab_service') as mock_cmab_service, \
1935+
mock.patch.object(self.project_config, 'get_variation_from_id',
1936+
return_value=entities.Variation('111151', 'variation_1')), \
1937+
mock.patch.object(self.decision_service, 'get_stored_variation') as mock_get_stored, \
1938+
mock.patch.object(self.decision_service,
1939+
'logger') as mock_logger:
1940+
1941+
mock_cmab_service.get_decision.return_value = (
1942+
{
1943+
'variation_id': '111151',
1944+
'cmab_uuid': 'test-cmab-uuid-123'
1945+
},
1946+
[]
1947+
)
1948+
1949+
variation_result = self.decision_service.get_variation(
1950+
self.project_config,
1951+
cmab_experiment,
1952+
user,
1953+
tracker
1954+
)
1955+
1956+
# Verify get_stored_variation was NOT called for CMAB experiment
1957+
mock_get_stored.assert_not_called()
1958+
1959+
# Verify the UPS skip reason is in the decision reasons
1960+
reasons = variation_result['reasons']
1961+
self.assertIn(
1962+
'Skipping user profile service for CMAB experiment "cmab_experiment".',
1963+
reasons
1964+
)
1965+
1966+
# Verify the variation was still resolved via CMAB service
1967+
self.assertEqual(entities.Variation('111151', 'variation_1'), variation_result['variation'])
1968+
self.assertEqual('test-cmab-uuid-123', variation_result['cmab_uuid'])
1969+
self.assertStrictFalse(variation_result['error'])
1970+
1971+
mock_logger.info.assert_any_call(
1972+
'Skipping user profile service for CMAB experiment "cmab_experiment".'
1973+
)
1974+
1975+
def test_get_variation_cmab_experiment_skips_ups_save(self):
1976+
"""Test that CMAB experiments do not save decisions to user profile service."""
1977+
1978+
user = optimizely_user_context.OptimizelyUserContext(
1979+
optimizely_client=None,
1980+
logger=None,
1981+
user_id="test_user",
1982+
user_attributes={}
1983+
)
1984+
1985+
cmab_experiment = entities.Experiment(
1986+
'111150',
1987+
'cmab_experiment',
1988+
'Running',
1989+
'111150',
1990+
[],
1991+
{},
1992+
[
1993+
entities.Variation('111151', 'variation_1'),
1994+
entities.Variation('111152', 'variation_2')
1995+
],
1996+
[
1997+
{'entityId': '111151', 'endOfRange': 5000},
1998+
{'entityId': '111152', 'endOfRange': 10000}
1999+
],
2000+
cmab={'trafficAllocation': 5000}
2001+
)
2002+
2003+
ups = user_profile.UserProfileService()
2004+
tracker = user_profile.UserProfileTracker("test_user", ups, self.decision_service.logger)
2005+
2006+
with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \
2007+
mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \
2008+
mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id',
2009+
return_value=['$', []]), \
2010+
mock.patch.object(self.decision_service, 'cmab_service') as mock_cmab_service, \
2011+
mock.patch.object(self.project_config, 'get_variation_from_id',
2012+
return_value=entities.Variation('111151', 'variation_1')), \
2013+
mock.patch.object(tracker, 'update_user_profile') as mock_update_profile, \
2014+
mock.patch.object(self.decision_service, 'logger'):
2015+
2016+
mock_cmab_service.get_decision.return_value = (
2017+
{
2018+
'variation_id': '111151',
2019+
'cmab_uuid': 'test-cmab-uuid-123'
2020+
},
2021+
[]
2022+
)
2023+
2024+
variation_result = self.decision_service.get_variation(
2025+
self.project_config,
2026+
cmab_experiment,
2027+
user,
2028+
tracker
2029+
)
2030+
2031+
# Verify update_user_profile was NOT called for CMAB experiment
2032+
mock_update_profile.assert_not_called()
2033+
2034+
# Verify the variation was still returned correctly
2035+
self.assertEqual(entities.Variation('111151', 'variation_1'), variation_result['variation'])
2036+
self.assertEqual('test-cmab-uuid-123', variation_result['cmab_uuid'])
2037+
self.assertStrictFalse(variation_result['error'])
2038+
2039+
def test_get_variation_non_cmab_experiment_uses_ups(self):
2040+
"""Test that non-CMAB experiments still use user profile service normally."""
2041+
2042+
user = optimizely_user_context.OptimizelyUserContext(
2043+
optimizely_client=None,
2044+
logger=None,
2045+
user_id="test_user",
2046+
user_attributes={}
2047+
)
2048+
2049+
# Create a non-CMAB experiment (cmab=None)
2050+
non_cmab_experiment = entities.Experiment(
2051+
'111127',
2052+
'test_experiment',
2053+
'Running',
2054+
'111182',
2055+
[],
2056+
{},
2057+
[
2058+
entities.Variation('111128', 'control'),
2059+
entities.Variation('111129', 'variation')
2060+
],
2061+
[
2062+
{'entityId': '111128', 'endOfRange': 5000},
2063+
{'entityId': '111129', 'endOfRange': 10000}
2064+
],
2065+
cmab=None
2066+
)
2067+
2068+
ups = user_profile.UserProfileService()
2069+
tracker = user_profile.UserProfileTracker("test_user", ups, self.decision_service.logger)
2070+
tracker.user_profile = user_profile.UserProfile(
2071+
"test_user",
2072+
{"111127": {"variation_id": "111128"}}
2073+
)
2074+
2075+
with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \
2076+
mock.patch.object(self.decision_service, 'get_stored_variation',
2077+
return_value=entities.Variation('111128', 'control')) as mock_get_stored, \
2078+
mock.patch.object(self.decision_service, 'logger') as mock_logger:
2079+
2080+
variation_result = self.decision_service.get_variation(
2081+
self.project_config,
2082+
non_cmab_experiment,
2083+
user,
2084+
tracker
2085+
)
2086+
2087+
# Verify get_stored_variation WAS called for non-CMAB experiment
2088+
mock_get_stored.assert_called_once()
2089+
2090+
# Verify the stored variation was returned
2091+
self.assertEqual(entities.Variation('111128', 'control'), variation_result['variation'])
2092+
self.assertIsNone(variation_result['cmab_uuid'])
2093+
self.assertStrictFalse(variation_result['error'])
2094+
2095+
# Verify no UPS skip message in reasons
2096+
reasons = variation_result['reasons']
2097+
for reason in reasons:
2098+
self.assertNotIn('Skipping user profile service', reason)

0 commit comments

Comments
 (0)