From ac37a98a142f8f9ca52f1c7622bf76f8ac5e5f14 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 5 Feb 2025 16:04:27 -0600 Subject: [PATCH 1/9] Rename transfer action providers As per https://docs.globus.org/api/transfer/action-providers/migration/ --- automate/minimus_mdf_flow.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/automate/minimus_mdf_flow.py b/automate/minimus_mdf_flow.py index 0ec44c1..9476c43 100644 --- a/automate/minimus_mdf_flow.py +++ b/automate/minimus_mdf_flow.py @@ -79,7 +79,7 @@ def file_transfer_steps(): "CreateDatasetDir": { "Comment": "Insure the dataset directory exists before attempting to create the version subdirectory", "Type": "Action", - "ActionUrl": "https://actions.globus.org/transfer/mkdir", + "ActionUrl": "https://transfer.actions.globus.org/mkdir", "ExceptionOnActionFailure": False, "Parameters": { "endpoint_id.$": "$.user_transfer_inputs.destination_endpoint_id", @@ -91,7 +91,7 @@ def file_transfer_steps(): "CreateDestinationDir": { "Comment": "Create a destination directory for the transferred data", "Type": "Action", - "ActionUrl": "https://actions.globus.org/transfer/mkdir", + "ActionUrl": "https://transfer.actions.globus.org/mkdir", "ExceptionOnActionFailure": True, "Parameters": { "endpoint_id.$": "$.user_transfer_inputs.destination_endpoint_id", @@ -114,7 +114,7 @@ def file_transfer_steps(): "UserPermissions": { "Comment": "Temporarily add write permissions for the submitting user", "Type": "Action", - "ActionUrl": "https://actions.globus.org/transfer/set_permission", + "ActionUrl": "https://transfer.actions.globus.org/set_permission", "ExceptionOnActionFailure": False, "Parameters": { "operation": "CREATE", @@ -141,14 +141,14 @@ def file_transfer_steps(): "UserTransfer": { "Comment": "Copy from user's endpoint to organization's dataset destination", "Type": "Action", - "ActionUrl": "https://actions.globus.org/transfer/transfer", + "ActionUrl": "https://transfer.actions.globus.org/transfer", "WaitTime": 86400, "RunAs": "SubmittingUser", "Parameters": { - "source_endpoint_id.$": "$.user_transfer_inputs.source_endpoint_id", - "destination_endpoint_id.$": "$.user_transfer_inputs.destination_endpoint_id", + "source_endpoint.$": "$.user_transfer_inputs.source_endpoint_id", + "destination_endpoint.$": "$.user_transfer_inputs.destination_endpoint_id", "label.$": "$.user_transfer_inputs.label", - "transfer_items.$": "$.user_transfer_inputs.transfer_items", + "DATA.$": "$.user_transfer_inputs.transfer_items", }, "ResultPath": "$.UserTransferResult", "Next": "UndoUserPermissions", @@ -156,7 +156,7 @@ def file_transfer_steps(): "UndoUserPermissions": { "Comment": "Remove temporary write permissions for the submitting user", "Type": "Action", - "ActionUrl": "https://actions.globus.org/transfer/set_permission", + "ActionUrl": "https://transfer.actions.globus.org/set_permission", "ExceptionOnActionFailure": False, "Parameters": { "operation": "DELETE", From 2d6d6483b953d4469b543a80ba77707384f857cc Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 5 Feb 2025 16:53:11 -0600 Subject: [PATCH 2/9] Fix set_permission rename --- automate/minimus_mdf_flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automate/minimus_mdf_flow.py b/automate/minimus_mdf_flow.py index 9476c43..adfd644 100644 --- a/automate/minimus_mdf_flow.py +++ b/automate/minimus_mdf_flow.py @@ -114,7 +114,7 @@ def file_transfer_steps(): "UserPermissions": { "Comment": "Temporarily add write permissions for the submitting user", "Type": "Action", - "ActionUrl": "https://transfer.actions.globus.org/set_permission", + "ActionUrl": "https://transfer.actions.globus.org/manage_permission", "ExceptionOnActionFailure": False, "Parameters": { "operation": "CREATE", From 44804f0673f8ba87aa308e60a28f5e0f7fbc40fe Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 5 Feb 2025 17:17:16 -0600 Subject: [PATCH 3/9] Fix manage_permissions AP --- automate/minimus_mdf_flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automate/minimus_mdf_flow.py b/automate/minimus_mdf_flow.py index adfd644..083b714 100644 --- a/automate/minimus_mdf_flow.py +++ b/automate/minimus_mdf_flow.py @@ -156,7 +156,7 @@ def file_transfer_steps(): "UndoUserPermissions": { "Comment": "Remove temporary write permissions for the submitting user", "Type": "Action", - "ActionUrl": "https://transfer.actions.globus.org/set_permission", + "ActionUrl": "https://transfer.actions.globus.org/manage_permission", "ExceptionOnActionFailure": False, "Parameters": { "operation": "DELETE", From 3204e5b05711a60238e1603cff42fab21aaa6cc6 Mon Sep 17 00:00:00 2001 From: Ben Blaiszik Date: Thu, 6 Feb 2025 14:59:19 -0600 Subject: [PATCH 4/9] Change all SubmittingUser to SubmittingUserV2 --- automate/create_new_flow.py | 4 ++-- automate/deploy_mdf_flow.py | 4 ++-- automate/minimus_mdf_flow.py | 2 +- aws/automate_manager.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/automate/create_new_flow.py b/automate/create_new_flow.py index c447aca..c527240 100644 --- a/automate/create_new_flow.py +++ b/automate/create_new_flow.py @@ -64,12 +64,12 @@ mdf_flow.save_flow("mdf_flow_info.prod.json") print("MDF Flow deployed", mdf_flow) -submitting_user_scope_id = mdf_flow.get_scope_id_for_runAs_role("SubmittingUser")[ +submitting_user_scope_id = mdf_flow.get_scope_id_for_runAs_role("SubmittingUserV2")[ "scopes" ][0]["id"] print(f"RunAs Dependent scope ID = {submitting_user_scope_id}") -submitting_user_scope_uri = mdf_flow.get_scope_uri_for_runAs_role("SubmittingUser") +submitting_user_scope_uri = mdf_flow.get_scope_uri_for_runAs_role("SubmittingUserV2") print( f"RunAs Dependent Scope URI (will appear in the dict of dependent scopes in the authorizer) = {submitting_user_scope_uri}" ) diff --git a/automate/deploy_mdf_flow.py b/automate/deploy_mdf_flow.py index 14528e6..199bffb 100644 --- a/automate/deploy_mdf_flow.py +++ b/automate/deploy_mdf_flow.py @@ -67,10 +67,10 @@ ])) mdf_flow.save_flow(flow_info_file) -print("scope = ", mdf_flow.get_scope_id_for_runAs_role('SubmittingUser')['scopes'][0]['id']) +print("scope = ", mdf_flow.get_scope_id_for_runAs_role('SubmittingUserV2')['scopes'][0]['id']) print("MDF Flow deployed", mdf_flow) -submitting_user_scope_id = mdf_flow.get_scope_id_for_runAs_role('SubmittingUser')['scopes'][0]['id'] +submitting_user_scope_id = mdf_flow.get_scope_id_for_runAs_role('SubmittingUserV2')['scopes'][0]['id'] connect_scope_def = { "scope": { diff --git a/automate/minimus_mdf_flow.py b/automate/minimus_mdf_flow.py index 083b714..896b2ae 100644 --- a/automate/minimus_mdf_flow.py +++ b/automate/minimus_mdf_flow.py @@ -143,7 +143,7 @@ def file_transfer_steps(): "Type": "Action", "ActionUrl": "https://transfer.actions.globus.org/transfer", "WaitTime": 86400, - "RunAs": "SubmittingUser", + "RunAs": "SubmittingUserV2", "Parameters": { "source_endpoint.$": "$.user_transfer_inputs.source_endpoint_id", "destination_endpoint.$": "$.user_transfer_inputs.destination_endpoint_id", diff --git a/aws/automate_manager.py b/aws/automate_manager.py index e923cfb..cd1b255 100644 --- a/aws/automate_manager.py +++ b/aws/automate_manager.py @@ -148,7 +148,7 @@ def submit(self, mdf_rec, organization, "datacite_prefix": self.datacite_prefix, "datacite_as_test": is_test, "_tokens": { - 'SubmittingUser': submitting_user_token['access_token'] + 'SubmittingUserV2': submitting_user_token['access_token'] } } From ff4b05235aca79d51ee85cf3198d41787966ac69 Mon Sep 17 00:00:00 2001 From: Ben Blaiszik Date: Thu, 6 Feb 2025 16:20:15 -0600 Subject: [PATCH 5/9] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index eff6fb2..eee1e10 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ respectively. The flow ID is stored in the `flow_id` key. You can verify deployment of the flows in the [Globus Automate Console](https://app.globus.org/flows/library). + ## Deploy the MDF Connect Service The MDF Connect service is deployed via a GitHub action. The action is triggered by a push to the dev or main branch. The action will deploy the service to the @@ -87,6 +88,11 @@ of the [Data Schemas Repo](https://github.com/materials-data-facility/data-schem The schema is deployed into the docker images used to serve up the lambda functions. +## Reviewing Logs +- [Dev Logs](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Faws$252Flambda$252FMDF-Connect2-submit-dev/log-events/) +- [Prod Logs](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Faws$252Flambda$252FMDF-Connect2-submit-prod/log-events/) + + # Running Tests To run the tests first make sure that you are running python 3.7.10. Then install the dependencies: From bfc2cddd40e87dfe8a347022e222bbe458e455c9 Mon Sep 17 00:00:00 2001 From: Ben Blaiszik Date: Thu, 6 Feb 2025 16:52:04 -0600 Subject: [PATCH 6/9] Update requirements-test.txt --- aws/tests/requirements-test.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/tests/requirements-test.txt b/aws/tests/requirements-test.txt index 7cc451b..5c7c13a 100644 --- a/aws/tests/requirements-test.txt +++ b/aws/tests/requirements-test.txt @@ -1,6 +1,6 @@ pytest<8.0 pytest-mock pytest-bdd==4.1.0 -git+https://github.com/materials-data-facility/connect_client.git@v0.4.0-dev +git+https://github.com/materials-data-facility/connect_client.git@v0.5.0 jsonschema>=2.6.0 -boto3 \ No newline at end of file +boto3 From 345b80edaf2961b4772cab61fc6f2ca0875dd87c Mon Sep 17 00:00:00 2001 From: Ben Blaiszik Date: Thu, 6 Feb 2025 16:55:25 -0600 Subject: [PATCH 7/9] Update requirements-test.txt --- aws/tests/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/tests/requirements-test.txt b/aws/tests/requirements-test.txt index 5c7c13a..0d7e94b 100644 --- a/aws/tests/requirements-test.txt +++ b/aws/tests/requirements-test.txt @@ -1,6 +1,6 @@ pytest<8.0 pytest-mock pytest-bdd==4.1.0 -git+https://github.com/materials-data-facility/connect_client.git@v0.5.0 +git+https://github.com/materials-data-facility/connect_client.git@0.5.0 jsonschema>=2.6.0 boto3 From e8dce9e442d44eb1a3db45d3f7e8dea7b0b06e13 Mon Sep 17 00:00:00 2001 From: Ben Blaiszik Date: Thu, 6 Feb 2025 17:00:41 -0600 Subject: [PATCH 8/9] Update requirements-test.txt --- aws/tests/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/tests/requirements-test.txt b/aws/tests/requirements-test.txt index 0d7e94b..6a3d642 100644 --- a/aws/tests/requirements-test.txt +++ b/aws/tests/requirements-test.txt @@ -1,6 +1,6 @@ pytest<8.0 pytest-mock pytest-bdd==4.1.0 -git+https://github.com/materials-data-facility/connect_client.git@0.5.0 +mdf-connect-client jsonschema>=2.6.0 boto3 From c9cfc8050978deb8d5203a6b72357aa7497ebd51 Mon Sep 17 00:00:00 2001 From: Ben Blaiszik Date: Thu, 6 Feb 2025 17:28:41 -0600 Subject: [PATCH 9/9] Update requirements-test.txt pytest-bdd --- aws/tests/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/tests/requirements-test.txt b/aws/tests/requirements-test.txt index 6a3d642..cef2139 100644 --- a/aws/tests/requirements-test.txt +++ b/aws/tests/requirements-test.txt @@ -1,6 +1,6 @@ pytest<8.0 pytest-mock -pytest-bdd==4.1.0 +pytest-bdd mdf-connect-client jsonschema>=2.6.0 boto3