From fd987ae8f927804e7396edb38f31f7c8128c18ce Mon Sep 17 00:00:00 2001 From: Laura Chevalier Date: Thu, 4 Jun 2026 15:38:28 +0000 Subject: [PATCH 1/5] Update experiments examples for clarity and accuracy. Change-Id: Ib200dcbfb2896e3712219e715ac39b310dc36cca --- .../create_asset_optimization_experiment.py | 2 +- .../create_search_adopt_ai_max_experiment.py | 4 +- .../create_search_custom_experiment.py | 14 +- ...e.py => evaluate_and_update_experiment.py} | 203 +++++++++++------- 4 files changed, 142 insertions(+), 81 deletions(-) rename examples/experiments/{get_experiment_performance.py => evaluate_and_update_experiment.py} (62%) diff --git a/examples/experiments/create_asset_optimization_experiment.py b/examples/experiments/create_asset_optimization_experiment.py index 1ab4dcb86..28db6cd7f 100644 --- a/examples/experiments/create_asset_optimization_experiment.py +++ b/examples/experiments/create_asset_optimization_experiment.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""This example shows how to create an OPTIMIZE_ASSETS experiment. +"""Creates an OPTIMIZE_ASSETS experiment. Asset optimization experiments are used to test different asset combinations within Performance Max campaigns. diff --git a/examples/experiments/create_search_adopt_ai_max_experiment.py b/examples/experiments/create_search_adopt_ai_max_experiment.py index 2e842a447..24963d4e9 100644 --- a/examples/experiments/create_search_adopt_ai_max_experiment.py +++ b/examples/experiments/create_search_adopt_ai_max_experiment.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""This example shows how to create an ADOPT_AI_MAX intra-campaign experiment for a Search campaign. +"""Creates an ADOPT_AI_MAX intra-campaign experiment for a Search campaign. Intra-campaign experiments split traffic *within* the campaign, based on whether the feature (in this case, AI Max) is enabled or not. @@ -75,6 +75,8 @@ def main(client: GoogleAdsClient, customer_id: str, campaign_id: str) -> None: # Create a campaign operation with an update mask to enable AI Max and # configure asset automation settings. + # Note: For intra-campaign experiments, these settings are applied to the + # base campaign but are only active for the treatment traffic split. campaign_operation = client.get_type("MutateOperation") campaign = campaign_operation.campaign_operation.update campaign.resource_name = googleads_service.campaign_path( diff --git a/examples/experiments/create_search_custom_experiment.py b/examples/experiments/create_search_custom_experiment.py index eaec7e9b5..a869ea57a 100644 --- a/examples/experiments/create_search_custom_experiment.py +++ b/examples/experiments/create_search_custom_experiment.py @@ -13,15 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""This example creates a standard, system-managed campaign experiment. +"""Creates a standard, system-managed campaign experiment of type SEARCH_CUSTOM. -It demonstrates how to create an experiment, configure its control and treatment -arms (where the treatment arm automatically generates a draft campaign), modify -the system-generated draft campaign, and schedule the experiment. +Sets up the experiment, configures control and treatment arms (where the +treatment arm automatically generates a draft campaign), modifies the +system-generated draft campaign, and schedules the experiment. -Note: This standard draft-based workflow does not apply to all experiment types -(e.g., intra-campaign or asset optimization experiments) that do not use system-generated -treatment campaign copies. +Note: This standard draft-based workflow applies only to experiment types +that use system-generated treatment campaign copies, and excludes +intra-campaign or asset optimization experiments. """ import argparse diff --git a/examples/experiments/get_experiment_performance.py b/examples/experiments/evaluate_and_update_experiment.py similarity index 62% rename from examples/experiments/get_experiment_performance.py rename to examples/experiments/evaluate_and_update_experiment.py index 08e9cdefc..ce245416a 100644 --- a/examples/experiments/get_experiment_performance.py +++ b/examples/experiments/evaluate_and_update_experiment.py @@ -12,10 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""This example illustrates how to retrieve performance metrics for an experiment. +"""Retrieves performance metrics for an experiment, evaluates the performance, +and takes action on the experiment accordingly. -It shows how to query statistical significance metrics for the experiment arms, -and how to execute actions such as promoting, ending, or graduating an experiment. +Queries statistical significance metrics for the experiment arms and executes +actions such as promoting, ending, or graduating an experiment. """ import argparse @@ -53,7 +54,11 @@ def main(client: GoogleAdsClient, customer_id: str, experiment_id: str) -> None: - """The main method that queries the experiment performance and evaluates it. + """Queries experiment performance, evaluates the performance metrics, and updates + the experiment accordingly (graduates, promotes, ends, or allows to continue running). + + While it retrieves metrics for all experiment arms, the evaluation logic focuses + on the treatment arms where statistical significance metrics are available. Args: client: an initialized GoogleAdsClient instance. @@ -62,9 +67,9 @@ def main(client: GoogleAdsClient, customer_id: str, experiment_id: str) -> None: """ ga_service: GoogleAdsServiceClient = client.get_service("GoogleAdsService") - # Query to retrieve both control and treatment arms under the parent experiment. - # Notice that we request the statistical metrics (e.g., p-value, point estimate, - # margin of error) which are populated exclusively on the treatment arm row. + # Query to retrieve all arms associated with the experiment. + # Note that statistical significance metrics (e.g., p-value, point estimate) + # are only populated on treatment arm rows, relative to the control arm. query = f""" SELECT experiment_arm.resource_name, @@ -101,8 +106,8 @@ def main(client: GoogleAdsClient, customer_id: str, experiment_id: str) -> None: # Statistical evaluation is only valid on the treatment (non-control) arm # because significance metrics are only populated relative to the baseline. - # Note: For intra-campaign/in-campaign experiments, only a single treatment row is - # returned (with control = False), since there is no separate control campaign. + # Note: For intra-campaign/in-campaign experiments, both control and treatment arm + # rows are returned, but evaluation is only performed on the treatment arm row. if not row.experiment_arm.control: evaluate_experiment(client, customer_id, row) @@ -110,17 +115,24 @@ def main(client: GoogleAdsClient, customer_id: str, experiment_id: str) -> None: print(f"No experiment arms found for experiment ID: {experiment_id}") -# [START get_experiment_performance_1] +# [START evaluate_and_update_experiment_1] def evaluate_experiment( client: GoogleAdsClient, customer_id: str, row: GoogleAdsRow ) -> None: - """Evaluates the performance of the treatment experiment arm. + """Evaluates the performance of the treatment experiment arm and updates + the experiment accordingly (e.g. promotes, ends, or graduates). + + Checks conversion and click metrics against statistical significance thresholds + to determine the appropriate action to take on the experiment. Args: client: an initialized GoogleAdsClient instance. customer_id: a client customer ID. row: a GoogleAdsRow containing the experiment arm and metrics. """ + # This function evaluates performance metrics and immediately takes action + # to update the experiment's status (promote, end, or graduate) if + # statistical significance thresholds are met. metrics = row.metrics experiment_resource_name = row.experiment.resource_name @@ -129,71 +141,107 @@ def evaluate_experiment( # - Margin of Error: Outlines the confidence interval bounds. Note that the margin_of_error provided by the API is calculated for a preset confidence level which is set based on the experiment type. # - Lower Bound: (Point Estimate - Margin of Error). If this value is above 0, # we have statistical significance that performance has improved. - conv_p_value = metrics.conversions_absolute_change_p_value - conv_lift = metrics.conversions_absolute_change_point_estimate - conv_error = metrics.conversions_absolute_change_margin_of_error - conv_lower_bound = conv_lift - conv_error + has_conv_metrics = ( + "conversions_absolute_change_p_value" in metrics + and "conversions_absolute_change_point_estimate" in metrics + and "conversions_absolute_change_margin_of_error" in metrics + ) + has_click_metrics = ( + "clicks_p_value" in metrics + and "clicks_point_estimate" in metrics + and "clicks_margin_of_error" in metrics + ) - if conv_p_value <= P_VALUE_THRESHOLD: - if conv_lower_bound > 0: + # 1. Evaluate conversion success as a primary success signal if available. + if has_conv_metrics: + conv_p_value = metrics.conversions_absolute_change_p_value + conv_lift = metrics.conversions_absolute_change_point_estimate + conv_error = metrics.conversions_absolute_change_margin_of_error + conv_lower_bound = conv_lift - conv_error + + if conv_p_value <= P_VALUE_THRESHOLD: + if conv_lower_bound > 0: + print( + "Significant Success: Conversions increased. Even at the lower" + f" bound, the lift is {conv_lower_bound:.2f}. Promoting" + " changes." + ) + promote_experiment( + client, customer_id, experiment_resource_name + ) + return + elif (conv_lift + conv_error) < 0: + print( + "Significant Decline: Even the upper bound" + f" ({conv_lift + conv_error:.2f}) is below zero. Ending" + " experiment." + ) + end_experiment(client, customer_id, experiment_resource_name) + return + + # 2. Evaluate click volume as a secondary signal. + # This is helpful as an early indicator or for lower-volume accounts. + click_p_value = metrics.clicks_p_value + click_lift = metrics.clicks_point_estimate + click_error = metrics.clicks_margin_of_error + click_lower_bound = click_lift - click_error + + if click_p_value <= P_VALUE_THRESHOLD and click_lower_bound > 0: + # We have a directional winner: high confidence in more traffic, + # but not enough data to confirm conversion impact yet. print( - "Significant Success: Conversions increased. Even at the lower" - f" bound, the lift is {conv_lower_bound:.2f}. Promoting" - " changes." + f"Click volume is significantly up (+{click_lift*100:.1f}%). " + "Graduating treatment for further manual analysis." ) - promote_experiment(client, customer_id, experiment_resource_name) - return - elif (conv_lift + conv_error) < 0: - print( - "Significant Decline: Even the upper bound" - f" ({conv_lift + conv_error:.2f}) is below zero. Ending" - " experiment." - ) - end_experiment(client, customer_id, experiment_resource_name) - return - # 2. Evaluate click volume as a secondary signal. - # This is helpful as an early indicator or for lower-volume accounts. - click_p_value = metrics.clicks_p_value - click_lift = metrics.clicks_point_estimate - click_error = metrics.clicks_margin_of_error - click_lower_bound = click_lift - click_error + # Graduate if it's a separate campaign test. + # This keeps the high-volume treatment running independently. + # Intra-campaign experiments (like ADOPT_BROAD_MATCH_KEYWORDS and + # ADOPT_AI_MAX) run directly within the base campaign, meaning there is only + # a single campaign involved and no separate treatment campaign to graduate. + # Therefore, graduation is not supported for intra-campaign experiments. + experiment_type_name = row.experiment.type_.name + if ( + experiment_type_name != "ADOPT_BROAD_MATCH_KEYWORDS" + and experiment_type_name != "ADOPT_AI_MAX" + ): + graduate_experiment( + client, customer_id, experiment_resource_name + ) + else: + print( + "Intra-campaign trial detected: Graduation is not supported" + " because there is only one campaign. Continuing to run to" + " gather more conversion data." + ) + return - if click_p_value <= P_VALUE_THRESHOLD and click_lower_bound > 0: - # We have a directional winner: high confidence in more traffic, - # but not enough data to confirm conversion impact yet. + # 3. Print status if no action was taken. + if has_conv_metrics or has_click_metrics: + conv_status = ( + f"Conversions (p={metrics.conversions_absolute_change_p_value:.2f}, " + f"lift={metrics.conversions_absolute_change_point_estimate:.2f} +/- " + f"{metrics.conversions_absolute_change_margin_of_error:.2f})" + if has_conv_metrics + else "Conversions (not populated)" + ) + click_status = ( + f"Clicks (p={metrics.clicks_p_value:.2f}, " + f"lift={metrics.clicks_point_estimate:.2f} +/- " + f"{metrics.clicks_margin_of_error:.2f})" + if has_click_metrics + else "Clicks (not populated)" + ) print( - f"Click volume is significantly up (+{click_lift*100:.1f}%). " - "Graduating treatment for further manual analysis." + f"Inconclusive: No significant action taken. {conv_status}, {click_status}." + " Continue running." ) - - # Graduate if it's a separate campaign test. - # This keeps the high-volume treatment running independently. - # Intra-campaign experiments (like ADOPT_BROAD_MATCH_KEYWORDS and - # ADOPT_AI_MAX) run directly within the base campaign, meaning there is only - # a single campaign involved and no separate treatment campaign to graduate. - # Therefore, graduation is not supported for intra-campaign experiments. - experiment_type_name = row.experiment.type_.name - if ( - experiment_type_name != "ADOPT_BROAD_MATCH_KEYWORDS" - and experiment_type_name != "ADOPT_AI_MAX" - ): - graduate_experiment(client, customer_id, experiment_resource_name) - else: - print( - "Intra-campaign trial detected: Graduation is not supported" - " because there is only one campaign. Continuing to run to" - " gather more conversion data." - ) else: - # Both conversions and clicks are noisy. print( - "Inconclusive: No significant lift in Conversions" - f" (p={conv_p_value:.2f}) or Clicks (p={click_p_value:.2f})." - f" Current estimated lift: {conv_lift:.2f} +/- {conv_error:.2f}." - " Continue running." + "Conversions and Clicks performance metrics are not yet populated. " + "Continue running." ) - # [END get_experiment_performance_1] + # [END evaluate_and_update_experiment_1] def promote_experiment( @@ -201,8 +249,9 @@ def promote_experiment( ) -> None: """Promotes the experiment trial campaign to the base campaign. - Promotion is an asynchronous long-running process that copies the trial campaign's - settings and creatives back to the base campaign and subsequently ends the experiment. + Promotion is an asynchronous long-running process that copies the trial + campaign's settings and creatives back to the base campaign and subsequently + ends the experiment. Args: client: an initialized GoogleAdsClient instance. @@ -213,6 +262,12 @@ def promote_experiment( "ExperimentService" ) # This method returns a long running operation (LRO). + # - To block until the operation is complete: call operation.result() + # - For non-blocking status checks: use operation.done() + # - For manual polling or persistent tracking: store operation.operation.name + # + # For more information on handling LROs, see: + # https://developers.google.com/google-ads/api/docs/concepts/long-running-operations operation = experiment_service.promote_experiment( resource_name=experiment_resource_name ) @@ -232,8 +287,7 @@ def end_experiment( ) -> None: """Immediately ends the experiment. - This sets the scheduled end date of the experiment to the current date and time, - terminating further traffic split serving without waiting for the end of the day. + Terminates the traffic split and sets the end date to the current time. Args: client: an initialized GoogleAdsClient instance. @@ -250,7 +304,9 @@ def end_experiment( def graduate_experiment( client: GoogleAdsClient, customer_id: str, experiment_resource_name: str ) -> None: - """Graduates the experiment to a full campaign. + """Graduates the experiment to a full standalone campaign. + + This process involves creating a new budget and mapping the treatment campaign to it. Args: client: an initialized GoogleAdsClient instance. @@ -281,6 +337,7 @@ def graduate_experiment( # 2. Query the experiment_arm to retrieve the treatment campaign's resource name. # The treatment arm has control set to False. ga_service: GoogleAdsServiceClient = client.get_service("GoogleAdsService") + # Query for the campaigns associated with the treatment arm of the experiment. query = f""" SELECT experiment_arm.campaigns @@ -290,12 +347,14 @@ def graduate_experiment( """ search_response = ga_service.search(customer_id=customer_id, query=query) + # Find the resource name of the treatment campaign. treatment_campaign_resource_name = None for row in search_response: if row.experiment_arm.campaigns: treatment_campaign_resource_name = row.experiment_arm.campaigns[0] break + # Verify that a treatment campaign was found. if not treatment_campaign_resource_name: print( "Could not find the treatment campaign associated with this" @@ -303,7 +362,7 @@ def graduate_experiment( ) return - # 3. Build the Graduation Mapping and execute. + # 3. Build the Graduation Mapping and execute the graduation request. experiment_service: ExperimentServiceClient = client.get_service( "ExperimentService" ) From fdbcb90c4b09c65959942cf42624a680beae977e Mon Sep 17 00:00:00 2001 From: Laura Chevalier Date: Thu, 4 Jun 2026 15:45:56 +0000 Subject: [PATCH 2/5] Additional comment updates Change-Id: Ibde9164ddc0273e90661bb0508ee819051bab0a8 --- .../experiments/evaluate_and_update_experiment.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/experiments/evaluate_and_update_experiment.py b/examples/experiments/evaluate_and_update_experiment.py index ce245416a..070d724dd 100644 --- a/examples/experiments/evaluate_and_update_experiment.py +++ b/examples/experiments/evaluate_and_update_experiment.py @@ -136,11 +136,6 @@ def evaluate_experiment( metrics = row.metrics experiment_resource_name = row.experiment.resource_name - # 1. Evaluate conversion success as a primary success signal. - # - Point Estimate: Represents the estimated average lift or difference in conversions. - # - Margin of Error: Outlines the confidence interval bounds. Note that the margin_of_error provided by the API is calculated for a preset confidence level which is set based on the experiment type. - # - Lower Bound: (Point Estimate - Margin of Error). If this value is above 0, - # we have statistical significance that performance has improved. has_conv_metrics = ( "conversions_absolute_change_p_value" in metrics and "conversions_absolute_change_point_estimate" in metrics @@ -153,6 +148,10 @@ def evaluate_experiment( ) # 1. Evaluate conversion success as a primary success signal if available. + # - Point Estimate: Represents the estimated average lift or difference in conversions. + # - Margin of Error: Outlines the confidence interval bounds. Note that the margin_of_error provided by the API is calculated for a preset confidence level which is set based on the experiment type. + # - Lower Bound: (Point Estimate - Margin of Error). If this value is above 0, + # we have statistical significance that performance has improved. if has_conv_metrics: conv_p_value = metrics.conversions_absolute_change_p_value conv_lift = metrics.conversions_absolute_change_point_estimate @@ -196,7 +195,7 @@ def evaluate_experiment( # Graduate if it's a separate campaign test. # This keeps the high-volume treatment running independently. - # Intra-campaign experiments (like ADOPT_BROAD_MATCH_KEYWORDS and + # Note that intra-campaign experiments (like ADOPT_BROAD_MATCH_KEYWORDS and # ADOPT_AI_MAX) run directly within the base campaign, meaning there is only # a single campaign involved and no separate treatment campaign to graduate. # Therefore, graduation is not supported for intra-campaign experiments. From 068df37b6ef93949666012bff2a3dd591ed40cf4 Mon Sep 17 00:00:00 2001 From: Laura Chevalier Date: Thu, 4 Jun 2026 16:28:21 +0000 Subject: [PATCH 3/5] Update graduation-related comments and print statements. Change-Id: Ic5c222a74ee69603cbc7c09a4abd7e53b397f293 --- .../evaluate_and_update_experiment.py | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/examples/experiments/evaluate_and_update_experiment.py b/examples/experiments/evaluate_and_update_experiment.py index 96dcc670e..5fd963b79 100644 --- a/examples/experiments/evaluate_and_update_experiment.py +++ b/examples/experiments/evaluate_and_update_experiment.py @@ -178,30 +178,25 @@ def evaluate_experiment( if click_p_value <= P_VALUE_THRESHOLD and click_lower_bound > 0: # We have a directional winner: high confidence in more traffic, # but not enough data to confirm conversion impact yet. - print( - f"Click volume is significantly up (+{click_lift*100:.1f}%). " - "Graduating treatment for further manual analysis." - ) - - # Graduate if it's a separate campaign test. - # This keeps the high-volume treatment running independently. - # Note that intra-campaign experiments (like ADOPT_BROAD_MATCH_KEYWORDS and - # ADOPT_AI_MAX) run directly within the base campaign, meaning there is only - # a single campaign involved and no separate treatment campaign to graduate. - # Therefore, graduation is not supported for intra-campaign experiments. + print(f"Click volume is significantly up (+{click_lift*100:.1f}%).") + + # Graduation is only supported for separate campaign experiments, not + # intra-campaign experiments where there is no separate treatment campaign. experiment_type_name = row.experiment.type_.name if ( experiment_type_name != "ADOPT_BROAD_MATCH_KEYWORDS" and experiment_type_name != "ADOPT_AI_MAX" ): + print( + "Graduating treatment campaign for further manual analysis." + ) graduate_experiment( client, customer_id, experiment_resource_name ) else: print( - "Intra-campaign trial detected: Graduation is not supported" - " because there is only one campaign. Continuing to run to" - " gather more conversion data." + "Intra-campaign trial detected: graduation is not supported. " + "Continuing to run the experiment to gather more conversion data." ) return From c1af3a4b230731d9932919231cc9c5257e9e9cb5 Mon Sep 17 00:00:00 2001 From: Laura Chevalier Date: Thu, 4 Jun 2026 16:31:21 +0000 Subject: [PATCH 4/5] Update casing. Change-Id: I5d2db1eb14c696f59e63e2b1d01241ba352f68cc --- examples/experiments/evaluate_and_update_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/experiments/evaluate_and_update_experiment.py b/examples/experiments/evaluate_and_update_experiment.py index 5fd963b79..d9eaa2e2a 100644 --- a/examples/experiments/evaluate_and_update_experiment.py +++ b/examples/experiments/evaluate_and_update_experiment.py @@ -222,7 +222,7 @@ def evaluate_experiment( ) else: print( - "Conversions and Clicks performance metrics are not yet populated. " + "Conversion and click performance metrics are not yet populated. " "Continue running." ) # [END evaluate_and_update_experiment_1] From a289f5a1963cb83b431e5bc85be3db35361d3f06 Mon Sep 17 00:00:00 2001 From: Laura Chevalier Date: Thu, 4 Jun 2026 19:54:00 +0000 Subject: [PATCH 5/5] Address comments Change-Id: I78ec54d77cc1d9f6f3797e6fdd9293f3a33adadf --- .../experiments/evaluate_and_update_experiment.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/experiments/evaluate_and_update_experiment.py b/examples/experiments/evaluate_and_update_experiment.py index d9eaa2e2a..ca0e43952 100644 --- a/examples/experiments/evaluate_and_update_experiment.py +++ b/examples/experiments/evaluate_and_update_experiment.py @@ -65,7 +65,7 @@ def main(client: GoogleAdsClient, customer_id: str, experiment_id: str) -> None: ga_service: GoogleAdsServiceClient = client.get_service("GoogleAdsService") # Query to retrieve the experiment. - # Notice that we request the statistical metrics (e.g., p-value, point estimate, + # Notice that we request the statistical metrics (for example, p-value, point estimate, # margin of error) which are populated based on the treatment arm. query = f""" SELECT @@ -110,7 +110,7 @@ def evaluate_experiment( client: GoogleAdsClient, customer_id: str, row: GoogleAdsRow ) -> None: """Evaluates the performance of the experiment and updates it accordingly - (e.g. promotes, ends, or graduates). + (for example, promotes, ends, or graduates). Checks conversion and click metrics against statistical significance thresholds to determine the appropriate action to take on the experiment. @@ -218,12 +218,12 @@ def evaluate_experiment( ) print( f"Inconclusive: No significant action taken. {conv_status}, {click_status}." - " Continue running." + " Allowing the experiment to continue running." ) else: print( "Conversion and click performance metrics are not yet populated. " - "Continue running." + "Allowing the experiment to continue running." ) # [END evaluate_and_update_experiment_1] @@ -346,7 +346,7 @@ def graduate_experiment( ) return - # 3. Build the Graduation Mapping and execute the graduation request. + # 3. Build the budget mapping and execute the graduation request. experiment_service: ExperimentServiceClient = client.get_service( "ExperimentService" ) @@ -370,7 +370,8 @@ def graduate_experiment( if __name__ == "__main__": parser = argparse.ArgumentParser( description=( - "Lists and evaluates performance metrics for a campaign experiment." + "Retrieves performance metrics for an experiment, evaluates the" + " performance and takes action on the experiment accordingly." ) ) # The following argument(s) should be provided to run the example.