From e1d5cdc868ddf00f82f51241f9bb47da5e9a9d99 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Tue, 13 Jun 2023 16:00:23 +0200 Subject: [PATCH 01/17] TECH-714: openimis added as example application --- examples/openIMIS/.env | 32 +++++++++++++++ examples/openIMIS/docker-compose.yml | 58 ++++++++++++++++++++++++++++ examples/openIMIS/import_data.py | 3 ++ examples/openIMIS/test_entrypoint.sh | 7 ++++ 4 files changed, 100 insertions(+) create mode 100644 examples/openIMIS/.env create mode 100644 examples/openIMIS/docker-compose.yml create mode 100755 examples/openIMIS/import_data.py create mode 100755 examples/openIMIS/test_entrypoint.sh diff --git a/examples/openIMIS/.env b/examples/openIMIS/.env new file mode 100644 index 0000000..2f61ce1 --- /dev/null +++ b/examples/openIMIS/.env @@ -0,0 +1,32 @@ +# database parameters +DB_HOST=db +DB_PORT=1433 +DB_PASSWORD=IMISuserP@s +DB_USER=IMISuser +DB_NAME=IMIS +SITE_ROOT=api +DB_ENGINE=mssql +DEBUG=True +HTTP_PORT=80 +ACCEPT_EULA=Y + +# url of openimis +# http port for openimis +GATEWAY_PORT=80 +# https port for openimis +GATEWAY_PORT_S=443 + +# github branches to use +# Database +#DB_BRANCH=main +#Gate way +#GW_BRANCH=main +# BE assembly +#BE_BRANCH=42q3g4f34r +# REST API +RESTAPI_BRANCH=main +# branch to get the code of the restapi +RESTAPI_BUILD_FLAVOR=Release + +PROJECT_NAME=openimis + diff --git a/examples/openIMIS/docker-compose.yml b/examples/openIMIS/docker-compose.yml new file mode 100644 index 0000000..c6d284e --- /dev/null +++ b/examples/openIMIS/docker-compose.yml @@ -0,0 +1,58 @@ +version: "2.4" + +services: + db: + container_name: ${PROJECT_NAME:-openimis}-db + image: ghcr.io/openimis/openimis-mssql:${DB_TAG:-latest} + restart: always + environment: + - DB_USER_PASSWORD=${DB_PASSWORD} + - SA_PASSWORD=${DB_PASSWORD} + - DB_NAME=${DB_NAME} + - DB_USER=${DB_USER} + - ACCEPT_EULA=${ACCEPT_EULA} + healthcheck: + test: "bash /app/healthcheck.sh" + interval: 10s + timeout: 5s + retries: 5 + start_period: 100s + networks: + openimis-net: + volumes: + - database:/var/opt/mssql/data + ports: + - 1433:1433 + backend: + build: + context: https://github.com/openimis/openimis-be_py.git#Test_Harness_Setup + args: + OPENIMIS_CONF_JSON: ${OPENIMIS_BE_CONF_JSON} + image: openimis/backend:${PROJECT_NAME:-local} + volumes: + - photos:/openimis-be/openIMIS/images/insurees + - ./import_data.py:/openimis-be/openIMIS/custom_scripts/import_data.py + command: start + environment: &backend-env + - DB_HOST=${DB_HOST} + - DB_PORT=${DB_PORT} + - DB_NAME=${DB_NAME} + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - SITE_ROOT=api + - SITE_URL=${NEW_OPENIMIS_HOST} + - CELERY_BROKER_URL=amqp://rabbitmq + - PHOTO_ROOT_PATH=images/insurees + depends_on: + db: + condition: service_healthy + networks: + openimis-net: + ports: + - 3333:8000 + +networks: + openimis-net: +volumes: + database: + photos: diff --git a/examples/openIMIS/import_data.py b/examples/openIMIS/import_data.py new file mode 100755 index 0000000..6838326 --- /dev/null +++ b/examples/openIMIS/import_data.py @@ -0,0 +1,3 @@ +# Note: This file will be used to hold a script +# that will be inserted into a Docker container with openIMIS. +# The script will take seed data and convert it into records in openIMIS. diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh new file mode 100755 index 0000000..e9db637 --- /dev/null +++ b/examples/openIMIS/test_entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +docker-compose up --build -d db backend + +# exec inserted script in the container +docker-compose exec backend python /openimis-be/openIMIS/custom_scripts/import_data.py + From d5680d06013a14830fc5f4c99cf8e9cf1ca11692 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Tue, 13 Jun 2023 16:40:52 +0200 Subject: [PATCH 02/17] TECH-714: openIMIS example app env changed --- examples/openIMIS/.env | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/examples/openIMIS/.env b/examples/openIMIS/.env index 2f61ce1..f4a51fb 100644 --- a/examples/openIMIS/.env +++ b/examples/openIMIS/.env @@ -1,3 +1,4 @@ +PROJECT_NAME=openimis # database parameters DB_HOST=db DB_PORT=1433 @@ -9,24 +10,5 @@ DB_ENGINE=mssql DEBUG=True HTTP_PORT=80 ACCEPT_EULA=Y - -# url of openimis -# http port for openimis -GATEWAY_PORT=80 -# https port for openimis -GATEWAY_PORT_S=443 - -# github branches to use -# Database -#DB_BRANCH=main -#Gate way -#GW_BRANCH=main -# BE assembly -#BE_BRANCH=42q3g4f34r -# REST API -RESTAPI_BRANCH=main -# branch to get the code of the restapi -RESTAPI_BUILD_FLAVOR=Release - -PROJECT_NAME=openimis +DB_BRANCH=main From d3c049bc463c3d9bcedeb411db012f4bc4a892e2 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 14 Jun 2023 14:04:44 +0200 Subject: [PATCH 03/17] TECH-714: changed docker-compose flow --- examples/openIMIS/test_entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh index e9db637..5b3b747 100755 --- a/examples/openIMIS/test_entrypoint.sh +++ b/examples/openIMIS/test_entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash - -docker-compose up --build -d db backend +docker-compose build db backend +docker-compose up db backend # exec inserted script in the container docker-compose exec backend python /openimis-be/openIMIS/custom_scripts/import_data.py From 94a8a8b1319f820a7131f28de99630750c89b940 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 14 Jun 2023 14:56:41 +0200 Subject: [PATCH 04/17] TECH-714: openIMIS backend runs with -d flag --- examples/openIMIS/test_entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh index 5b3b747..b36d3ed 100755 --- a/examples/openIMIS/test_entrypoint.sh +++ b/examples/openIMIS/test_entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash docker-compose build db backend -docker-compose up db backend +docker-compose up -d db backend # exec inserted script in the container docker-compose exec backend python /openimis-be/openIMIS/custom_scripts/import_data.py From e16a3936dcafe25734ae1d2e674fc9d9b2434729 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 14 Jun 2023 15:25:29 +0200 Subject: [PATCH 05/17] TECH-714: 5s to 10 min wait for example API --- test/openAPI/docker/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/openAPI/docker/entrypoint.sh b/test/openAPI/docker/entrypoint.sh index a1f142a..01ffe69 100755 --- a/test/openAPI/docker/entrypoint.sh +++ b/test/openAPI/docker/entrypoint.sh @@ -40,9 +40,9 @@ waitForAPI() { healthcheckApiCall notAvailable=$? retries=$(( $retries - 1 )) - if [ $retries -ne 5 ] + if [ $retries -ne 10 ] then - sleep 1 + sleep 60 fi done From 205734c359fcdad265cedae8c4c27484b65570d9 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 14 Jun 2023 16:39:36 +0200 Subject: [PATCH 06/17] TECH-714: added check for service avability --- .../openIMIS/check_service_availability.sh | 29 +++++++++++++++++++ examples/openIMIS/docker-compose.yml | 1 + examples/openIMIS/test_entrypoint.sh | 3 +- 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100755 examples/openIMIS/check_service_availability.sh diff --git a/examples/openIMIS/check_service_availability.sh b/examples/openIMIS/check_service_availability.sh new file mode 100755 index 0000000..3c8b00c --- /dev/null +++ b/examples/openIMIS/check_service_availability.sh @@ -0,0 +1,29 @@ +#!/bin/bash +apt-get install -y netcat + +check_service() { + local service=$1 + local port=$2 + local max_retries=10 + local try=0 + + while [ $try -lt $max_retries ]; do + nc -z $service $port + result=$? + + if [ $result -eq 0 ]; then + echo "$service is up!" + return 0 + else + echo "Waiting for $service..." + sleep 50 + fi + + try=$((try + 1)) + done + + echo "$service failed to start in time." + return 1 +} + +check_service backend 8000 \ No newline at end of file diff --git a/examples/openIMIS/docker-compose.yml b/examples/openIMIS/docker-compose.yml index c6d284e..d707ca0 100644 --- a/examples/openIMIS/docker-compose.yml +++ b/examples/openIMIS/docker-compose.yml @@ -32,6 +32,7 @@ services: volumes: - photos:/openimis-be/openIMIS/images/insurees - ./import_data.py:/openimis-be/openIMIS/custom_scripts/import_data.py + - ./check_service_availability.sh:/openimis-be/openIMIS/custom_scripts/check_service_availability.sh command: start environment: &backend-env - DB_HOST=${DB_HOST} diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh index b36d3ed..b5b0761 100755 --- a/examples/openIMIS/test_entrypoint.sh +++ b/examples/openIMIS/test_entrypoint.sh @@ -2,6 +2,7 @@ docker-compose build db backend docker-compose up -d db backend -# exec inserted script in the container +docker-compose exec backend chmod +x /openimis-be/openIMIS/custom_scripts/check_service_availability.sh +docker-compose exec backend bash /openimis-be/openIMIS/custom_scripts/check_service_availability.sh docker-compose exec backend python /openimis-be/openIMIS/custom_scripts/import_data.py From 9e62553e323bb75dda08957e51b2472089ad95c5 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 14 Jun 2023 16:40:03 +0200 Subject: [PATCH 07/17] TECH-714: added check for service avability --- test/openAPI/docker/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/openAPI/docker/entrypoint.sh b/test/openAPI/docker/entrypoint.sh index 01ffe69..a1f142a 100755 --- a/test/openAPI/docker/entrypoint.sh +++ b/test/openAPI/docker/entrypoint.sh @@ -40,9 +40,9 @@ waitForAPI() { healthcheckApiCall notAvailable=$? retries=$(( $retries - 1 )) - if [ $retries -ne 10 ] + if [ $retries -ne 5 ] then - sleep 60 + sleep 1 fi done From 092b4b21fe65401ad69db9e67c43962f5fd171f8 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 14 Jun 2023 16:48:51 +0200 Subject: [PATCH 08/17] TECH-714: added blank line for check service script --- examples/openIMIS/check_service_availability.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/openIMIS/check_service_availability.sh b/examples/openIMIS/check_service_availability.sh index 3c8b00c..e6afee9 100755 --- a/examples/openIMIS/check_service_availability.sh +++ b/examples/openIMIS/check_service_availability.sh @@ -26,4 +26,4 @@ check_service() { return 1 } -check_service backend 8000 \ No newline at end of file +check_service backend 8000 From 175b71a173e762c3de57c71c3258cc2d1f55d5e5 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Thu, 15 Jun 2023 17:07:59 +0200 Subject: [PATCH 09/17] TECH-714: changed db to postgresql --- examples/openIMIS/.env | 11 ++++---- examples/openIMIS/docker-compose.yml | 39 +++++++++++++--------------- examples/openIMIS/test_entrypoint.sh | 9 +++---- 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/examples/openIMIS/.env b/examples/openIMIS/.env index f4a51fb..f767329 100644 --- a/examples/openIMIS/.env +++ b/examples/openIMIS/.env @@ -1,14 +1,13 @@ -PROJECT_NAME=openimis +PROJECT_NAME=digitalRegistryOpenIMIS # database parameters DB_HOST=db -DB_PORT=1433 +DB_PORT=5432 DB_PASSWORD=IMISuserP@s DB_USER=IMISuser DB_NAME=IMIS -SITE_ROOT=api -DB_ENGINE=mssql +DB_ENGINE=django.db.backends.postgresql DEBUG=True -HTTP_PORT=80 -ACCEPT_EULA=Y DB_BRANCH=main +SITE_ROOT=api +HTTP_PORT=80 diff --git a/examples/openIMIS/docker-compose.yml b/examples/openIMIS/docker-compose.yml index d707ca0..7ffa5a6 100644 --- a/examples/openIMIS/docker-compose.yml +++ b/examples/openIMIS/docker-compose.yml @@ -3,47 +3,44 @@ version: "2.4" services: db: container_name: ${PROJECT_NAME:-openimis}-db - image: ghcr.io/openimis/openimis-mssql:${DB_TAG:-latest} - restart: always + image: ghcr.io/openimis/openimis-pgsql:${DB_TAG:-latest} environment: - - DB_USER_PASSWORD=${DB_PASSWORD} - - SA_PASSWORD=${DB_PASSWORD} - - DB_NAME=${DB_NAME} - - DB_USER=${DB_USER} - - ACCEPT_EULA=${ACCEPT_EULA} + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=${DB_NAME} + - POSTGRES_USER=${DB_USER} healthcheck: - test: "bash /app/healthcheck.sh" + test: pg_isready -U ${DB_USER} -d ${DB_NAME} interval: 10s timeout: 5s retries: 5 start_period: 100s networks: openimis-net: - volumes: - - database:/var/opt/mssql/data ports: - - 1433:1433 + - 5433:5432 + backend: - build: - context: https://github.com/openimis/openimis-be_py.git#Test_Harness_Setup - args: - OPENIMIS_CONF_JSON: ${OPENIMIS_BE_CONF_JSON} + container_name: ${PROJECT_NAME:-openimis}-api image: openimis/backend:${PROJECT_NAME:-local} - volumes: - - photos:/openimis-be/openIMIS/images/insurees - - ./import_data.py:/openimis-be/openIMIS/custom_scripts/import_data.py - - ./check_service_availability.sh:/openimis-be/openIMIS/custom_scripts/check_service_availability.sh + restart: always command: start + build: + context: https://github.com/openimis/openimis-be_py.git#govstack-testing-setup + volumes: + - ./import_data.py:/import_data.py + - ./check_service_availability.sh:/check_service_availability.sh environment: &backend-env - DB_HOST=${DB_HOST} - - DB_PORT=${DB_PORT} + - DB_PORT=5432 - DB_NAME=${DB_NAME} - DB_USER=${DB_USER} + - DB_ENGINE=${DB_ENGINE} - DB_PASSWORD=${DB_PASSWORD} - SITE_ROOT=api + - DEBUG=True + - DJANGO_LOG_LEVEL=DEBUG - SITE_URL=${NEW_OPENIMIS_HOST} - CELERY_BROKER_URL=amqp://rabbitmq - - PHOTO_ROOT_PATH=images/insurees depends_on: db: condition: service_healthy diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh index b5b0761..6484d76 100755 --- a/examples/openIMIS/test_entrypoint.sh +++ b/examples/openIMIS/test_entrypoint.sh @@ -1,8 +1,7 @@ #!/bin/bash docker-compose build db backend -docker-compose up -d db backend - -docker-compose exec backend chmod +x /openimis-be/openIMIS/custom_scripts/check_service_availability.sh -docker-compose exec backend bash /openimis-be/openIMIS/custom_scripts/check_service_availability.sh -docker-compose exec backend python /openimis-be/openIMIS/custom_scripts/import_data.py +docker-compose up db backend +docker-compose exec backend chmod +x /check_service_availability.sh +docker-compose exec backend bash /check_service_availability.sh +docker-compose exec backend python /import_data.py From 2e3bb61fb539420b80391a6c61527d4f671cb994 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Thu, 15 Jun 2023 17:29:01 +0200 Subject: [PATCH 10/17] TECH-714: added -d flag --- examples/openIMIS/test_entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh index 6484d76..0dae5f1 100755 --- a/examples/openIMIS/test_entrypoint.sh +++ b/examples/openIMIS/test_entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash docker-compose build db backend -docker-compose up db backend +docker-compose up -d db backend docker-compose exec backend chmod +x /check_service_availability.sh docker-compose exec backend bash /check_service_availability.sh From 7c0a2750ae8a85687974308cc981736a8904ca80 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Tue, 20 Jun 2023 20:36:51 +0200 Subject: [PATCH 11/17] TECH-717: check for test flow --- examples/openIMIS/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/openIMIS/docker-compose.yml b/examples/openIMIS/docker-compose.yml index 7ffa5a6..ec67f56 100644 --- a/examples/openIMIS/docker-compose.yml +++ b/examples/openIMIS/docker-compose.yml @@ -29,6 +29,7 @@ services: volumes: - ./import_data.py:/import_data.py - ./check_service_availability.sh:/check_service_availability.sh + - ../../test/openAPI/test-data.json:/test-data.json environment: &backend-env - DB_HOST=${DB_HOST} - DB_PORT=5432 From 756d71c907447527cbf6e00aa4c3f6b7a20f1482 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Tue, 20 Jun 2023 21:19:13 +0200 Subject: [PATCH 12/17] TECH-717: set record UUID to match limit of 16 chars --- test/openAPI/test-data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/openAPI/test-data.json b/test/openAPI/test-data.json index 37e0c0d..eb83e44 100644 --- a/test/openAPI/test-data.json +++ b/test/openAPI/test-data.json @@ -57,7 +57,7 @@ "PersonalData": [] }, { - "uuid": "80bcc268-fa17-11ed-be56-0242ac1200022", + "uuid": "80bcc268-fa17-11ed-be56-0242ac120002", "id": "EE6546273487865", "FirstName": "Tobias", "LastName": "Smith", From da6396fc5ee5b4c4aa7a3197dc476e30754ddd39 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 21 Jun 2023 13:09:10 +0200 Subject: [PATCH 13/17] TECH-717: draft spacing --- examples/openIMIS/import_data.py | 86 ++++++++++++++++++++++++++++ examples/openIMIS/test_entrypoint.sh | 1 + 2 files changed, 87 insertions(+) diff --git a/examples/openIMIS/import_data.py b/examples/openIMIS/import_data.py index 6838326..a68572a 100755 --- a/examples/openIMIS/import_data.py +++ b/examples/openIMIS/import_data.py @@ -1,3 +1,89 @@ # Note: This file will be used to hold a script # that will be inserted into a Docker container with openIMIS. # The script will take seed data and convert it into records in openIMIS. + +import os +import django +import json + +# w main zrobić setup + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openimis-be.openIMIS.openIMIS.settings") +django.setup() +from core.models import Officer, InteractiveUser, User, TechnicalUser +from core.services import create_or_update_user_roles + +# from core.test_helpers import create_test_interactive_user + + +def load_django_settings(): + pass + +def load_json_data(file_path): + with open(file_path, 'r') as file: + return json.load(file) + + +def create_interactive_user_from_data( + user_id, + user_uuid, + username=None, + other_names=None, + last_name=None, + birth_certificate_id=None, + password="Test1234", + roles=None, + custom_props=None +): + if roles is None: + roles = [3] + if username is None: + username = other_names + + + # id musi być przechowywane w jsonExt, bo typ danych się różni + # ew. zmienić typ testowych danych na integer + i_user = InteractiveUser.objects.create( + **{ + "language_id": "en", + "uuid": user_uuid, + "last_name": last_name, + "other_names": other_names, + "login_name": username, + "audit_user_id": -1, + "role_id": roles[0] + } + ) + i_user.set_password(password) + i_user.save() + create_or_update_user_roles(i_user, roles, None) + return User.objects.create( + username=username, + i_user=i_user, + ) + + +if __name__ == "__main__": + print("It does work") + load_django_settings + test_data = load_json_data("../../test-data.json") + + print("-------------") + for record in test_data["registries"]["records"]: + print(record) + create_interactive_user_from_data( + user_id=record["id"], + user_uuid=record["uuid"], + username=record["FirstName"], + other_names=record["FirstName"], + last_name=record["LastName"], + birth_certificate_id=None, + password="Test1234", + roles=None, + custom_props=None + ) + print(record["id"], record["uuid"], record["FirstName"], record["FirstName"], record["LastName"]) + print("------------- end") + + # create_test_interactive_user("Lucas2") + print("report_definitions") \ No newline at end of file diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh index 0dae5f1..9e89d09 100755 --- a/examples/openIMIS/test_entrypoint.sh +++ b/examples/openIMIS/test_entrypoint.sh @@ -1,5 +1,6 @@ #!/bin/bash docker-compose build db backend + docker-compose up -d db backend docker-compose exec backend chmod +x /check_service_availability.sh From 953683c4bc0766475e473432e7812dc6d2ff7f46 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 21 Jun 2023 13:16:15 +0200 Subject: [PATCH 14/17] TECH-717: added no cache flag --- examples/openIMIS/test_entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh index 9e89d09..3f48cbb 100755 --- a/examples/openIMIS/test_entrypoint.sh +++ b/examples/openIMIS/test_entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/bash -docker-compose build db backend +docker-compose build --no-cache db backend docker-compose up -d db backend From 6163af9fd67d1b8b6459f75650ea056147da2e07 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 21 Jun 2023 13:27:20 +0200 Subject: [PATCH 15/17] TECH-717: disabled adding test data --- examples/openIMIS/import_data.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/openIMIS/import_data.py b/examples/openIMIS/import_data.py index a68572a..3d5dd83 100755 --- a/examples/openIMIS/import_data.py +++ b/examples/openIMIS/import_data.py @@ -71,17 +71,17 @@ def create_interactive_user_from_data( print("-------------") for record in test_data["registries"]["records"]: print(record) - create_interactive_user_from_data( - user_id=record["id"], - user_uuid=record["uuid"], - username=record["FirstName"], - other_names=record["FirstName"], - last_name=record["LastName"], - birth_certificate_id=None, - password="Test1234", - roles=None, - custom_props=None - ) + # create_interactive_user_from_data( + # user_id=record["id"], + # user_uuid=record["uuid"], + # username=record["FirstName"], + # other_names=record["FirstName"], + # last_name=record["LastName"], + # birth_certificate_id=None, + # password="Test1234", + # roles=None, + # custom_props=None + # ) print(record["id"], record["uuid"], record["FirstName"], record["FirstName"], record["LastName"]) print("------------- end") From 4cd1ced770b2fad5543521b36f4fc8752507c8e9 Mon Sep 17 00:00:00 2001 From: lruzicki Date: Wed, 21 Jun 2023 13:35:10 +0200 Subject: [PATCH 16/17] TECH-717: disabled whole script for tests --- examples/openIMIS/import_data.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/openIMIS/import_data.py b/examples/openIMIS/import_data.py index 3d5dd83..dfa117b 100755 --- a/examples/openIMIS/import_data.py +++ b/examples/openIMIS/import_data.py @@ -65,12 +65,12 @@ def create_interactive_user_from_data( if __name__ == "__main__": print("It does work") - load_django_settings - test_data = load_json_data("../../test-data.json") + # load_django_settings + # test_data = load_json_data("../../test-data.json") print("-------------") - for record in test_data["registries"]["records"]: - print(record) + # for record in test_data["registries"]["records"]: + # print(record) # create_interactive_user_from_data( # user_id=record["id"], # user_uuid=record["uuid"], @@ -82,8 +82,8 @@ def create_interactive_user_from_data( # roles=None, # custom_props=None # ) - print(record["id"], record["uuid"], record["FirstName"], record["FirstName"], record["LastName"]) - print("------------- end") + # print(record["id"], record["uuid"], record["FirstName"], record["FirstName"], record["LastName"]) + # print("------------- end") # create_test_interactive_user("Lucas2") - print("report_definitions") \ No newline at end of file + # print("report_definitions") \ No newline at end of file From 6216f1f96ad132baec0742d544d4d16205aad344 Mon Sep 17 00:00:00 2001 From: lruzicki <56487722+lruzicki@users.noreply.github.com> Date: Thu, 10 Aug 2023 14:53:11 +0200 Subject: [PATCH 17/17] Tech 760: custom parameters for test suite (#45) * TECH-760: added flow for custom parameters for gherkin tests * TECH-760: added custom ID for single scenario for test suite * TECH-760: edited script to change report templates * TECH-760: refactored parameters functions to be available in helpers js * TECH-760: when step updated with replacement function * TECH-760: deleted doubled step from tests * TECH-760: fixed typo in gherkin file * TECH-760: moved function to helpers js * TECH-760: extended test data with custom parameters * TECH-760: updated test default parameters * TECH-716: import data script added (#46) * TECH-716: import data script added * Update import_data.py Delete unsued function --------- Co-authored-by: lruzicki --------- Co-authored-by: lruzicki --- .circleci/config.yml | 9 +- .../openIMIS/createTestCustomParameters.py | 61 +++++++ examples/openIMIS/docker-compose.yml | 1 + examples/openIMIS/import_data.py | 168 +++++++++++------- examples/openIMIS/test_entrypoint.sh | 6 +- test/openAPI/features/data_create.feature | 14 +- test/openAPI/features/support/data_create.js | 19 +- .../features/support/helpers/helpers.js | 23 +++ test/openAPI/gherkin_param_manager.sh | 37 ++++ test/openAPI/test-data.json | 5 +- test/openAPI/testDefaultParameters.json | 54 ++++++ 11 files changed, 306 insertions(+), 91 deletions(-) create mode 100644 examples/openIMIS/createTestCustomParameters.py create mode 100755 test/openAPI/gherkin_param_manager.sh create mode 100644 test/openAPI/testDefaultParameters.json diff --git a/.circleci/config.yml b/.circleci/config.yml index a1e1889..bb8d894 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,18 +2,17 @@ version: 2.1 setup: true orbs: - test-harness: govstack-working-group/testutils@1.0.3 + testutils: govstack-working-group/testutils@dev:alpha workflows: test_everything: jobs: - - test-harness/create-config: + - testutils/create-config: post-steps: # Persist to workspace has to be defined in main workflow - persist_to_workspace: root: workspace paths: - generated.yml - - test-harness/execute-tests: + - testutils/execute-tests: requires: - - test-harness/create-config - + - testutils/create-config \ No newline at end of file diff --git a/examples/openIMIS/createTestCustomParameters.py b/examples/openIMIS/createTestCustomParameters.py new file mode 100644 index 0000000..2a08f2a --- /dev/null +++ b/examples/openIMIS/createTestCustomParameters.py @@ -0,0 +1,61 @@ +import requests +import json + +class CustomParamManager: + def __init__(self): + self.params = {} + + def fetch_tokens(self): + response = requests.post("http://localhost:3333/login/") + response_data = response.json() + self.params["csrf_token"] = response_data["csrf_token"] + self.params["jwt_token"] = response_data["jwt_token"] + + def add_params(self, new_params_list): + for new_params in new_params_list: + self.params.update(new_params) + + def save_to_file(self): + with open("testCustomParameters.json", "w") as json_file: + json.dump(self.params, json_file) + + def show_params(self): + print("Params:", self.params) + + +if __name__ == "__main__": + print("Creating custom parameters for test suite") + param_manager = CustomParamManager() + # param_manager.fetch_tokens() + + id_parameters = [{"ID_" + str(i): str(2000 + i - 1)} for i in range(1, 35)] + + first_name_parameters = [ + {"FirstName_0": "Anna"}, + {"FirstName_1": "Zofia"}, + {"FirstName_2": "Elsa"}, + {"FirstName_3": "Nick"}, + {"FirstName_4": "Thomas"}, + ] + + last_name_parameters = [ + {"LastName_0": "Stock"}, + {"LastName_1": "Don"}, + {"LastName_2": "Smith"}, + {"LastName_3": "Evans"}, + {"LastName_4": "Johnson"}, + ] + + birth_certificate_id = [ + {"BirthCertificateID_0": "RR-1234567999"}, + {"BirthCertificateID_1": "RR-5465679229"}, + {"BirthCertificateID_2": "RR-1234537999"}, + {"BirthCertificateID_3": "RR-7898797999"}, + {"BirthCertificateID_4": "RR-8534567781"}, + ] + + param_manager.add_params(id_parameters) + param_manager.add_params(first_name_parameters) + param_manager.add_params(last_name_parameters) + param_manager.add_params(birth_certificate_id) + param_manager.save_to_file() diff --git a/examples/openIMIS/docker-compose.yml b/examples/openIMIS/docker-compose.yml index ec67f56..bd105f4 100644 --- a/examples/openIMIS/docker-compose.yml +++ b/examples/openIMIS/docker-compose.yml @@ -28,6 +28,7 @@ services: context: https://github.com/openimis/openimis-be_py.git#govstack-testing-setup volumes: - ./import_data.py:/import_data.py + - ./createTestCustomParameters.py:/createTestCustomParameters.py - ./check_service_availability.sh:/check_service_availability.sh - ../../test/openAPI/test-data.json:/test-data.json environment: &backend-env diff --git a/examples/openIMIS/import_data.py b/examples/openIMIS/import_data.py index dfa117b..7b1cb14 100755 --- a/examples/openIMIS/import_data.py +++ b/examples/openIMIS/import_data.py @@ -1,89 +1,125 @@ -# Note: This file will be used to hold a script -# that will be inserted into a Docker container with openIMIS. -# The script will take seed data and convert it into records in openIMIS. - import os import django import json -# w main zrobić setup - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openimis-be.openIMIS.openIMIS.settings") django.setup() -from core.models import Officer, InteractiveUser, User, TechnicalUser -from core.services import create_or_update_user_roles -# from core.test_helpers import create_test_interactive_user +from insuree.gql_mutations import update_or_create_gender +from govstack_api.models import Registry +from insuree.models import Insuree, Family, Gender +from django.db import IntegrityError -def load_django_settings(): - pass - def load_json_data(file_path): with open(file_path, 'r') as file: return json.load(file) -def create_interactive_user_from_data( - user_id, - user_uuid, - username=None, - other_names=None, - last_name=None, - birth_certificate_id=None, - password="Test1234", - roles=None, - custom_props=None -): - if roles is None: - roles = [3] - if username is None: - username = other_names - - - # id musi być przechowywane w jsonExt, bo typ danych się różni - # ew. zmienić typ testowych danych na integer - i_user = InteractiveUser.objects.create( +def load_parameter_from_custom_parameters(file_path, parameter: str = ""): + with open(file_path, 'r') as file: + data = json.load(file) + return {k: v for k, v in data.items() if k.startswith(f'{parameter}_')} + + +def create_test_insuree( + with_family=True, is_head=False, custom_props=None, family_custom_props=None, + last_name=None, other_names=None, insuree_id=None, insuree_uuid=None, json_ext=None): + # insuree has a mandatory reference to family and family has a mandatory reference to insuree + # So we first insert the family with a dummy id and then update it + print(insuree_uuid) + if with_family: + family = Family.objects.create( + validity_from="2019-01-01", + head_insuree_id=1, # dummy + audit_user_id=-1, + **(family_custom_props if family_custom_props else {}) + ) + else: + family = None + + insuree = Insuree.objects.create( **{ - "language_id": "en", - "uuid": user_uuid, "last_name": last_name, + "id": insuree_id, + "uuid": insuree_uuid, "other_names": other_names, - "login_name": username, + "chf_id": str(insuree_id), + "family": Family.objects.get(id=1), + "gender": Gender.objects.get(code='M'), + "dob": "1920-04-02", + "head": is_head, + "card_issued": False, + "validity_from": "2019-01-01", "audit_user_id": -1, - "role_id": roles[0] + "json_ext": json_ext, + **(custom_props if custom_props else {}) } ) - i_user.set_password(password) - i_user.save() - create_or_update_user_roles(i_user, roles, None) - return User.objects.create( - username=username, - i_user=i_user, - ) + if with_family: + family.head_insuree_id = insuree.id + if family_custom_props: + for k, v in family_custom_props.items(): + setattr(family, k, v) + family.save() + + insuree.save() + return insuree + + +def create_gender(user, data): + update_or_create_gender(data, user) + + +def create_default_registry(registry_name=None, version=None, class_name=None, model=None, + fields_mapping=None, special_fields=None, default_values=None, + mutations=None, queries=None): + default_args = { + "registry_name": 'registryname', + "version": '111', + "class_name": 'InsureeRegistry', + "model": 'Insuree', + "fields_mapping": {"ID": "id", "uuid": "uuid", "LastName": "lastName", "FirstName": "otherNames"}, + "special_fields": ["BirthCertificateID", "PersonalData"], + "default_values": { + "dob": 'dob: "1920-04-02"', + "head": "head: false", + "family_id": "familyId: 1", + "gender_id": "genderId: \"M\"", + "card_issued": "cardIssued: false" + }, + "mutations": {"create": "createInsuree", "delete": "deleteInsurees", "update": "updateInsuree"}, + "queries": {"get": "insurees"} + } + + registry = Registry.objects.create( + registry_name=registry_name if registry_name is not None else default_args["registry_name"], + version=version if version is not None else default_args["version"], + class_name=class_name if class_name is not None else default_args["class_name"], + model=model if model is not None else default_args["model"], + fields_mapping=fields_mapping if fields_mapping is not None else default_args["fields_mapping"], + special_fields=special_fields if special_fields is not None else default_args["special_fields"], + default_values=default_values if default_values is not None else default_args["default_values"], + mutations=mutations if mutations is not None else default_args["mutations"], + queries=queries if queries is not None else default_args["queries"] + ) + registry.save() + print("Registry has been successfully saved to the database") if __name__ == "__main__": - print("It does work") - # load_django_settings - # test_data = load_json_data("../../test-data.json") - - print("-------------") - # for record in test_data["registries"]["records"]: - # print(record) - # create_interactive_user_from_data( - # user_id=record["id"], - # user_uuid=record["uuid"], - # username=record["FirstName"], - # other_names=record["FirstName"], - # last_name=record["LastName"], - # birth_certificate_id=None, - # password="Test1234", - # roles=None, - # custom_props=None - # ) - # print(record["id"], record["uuid"], record["FirstName"], record["FirstName"], record["LastName"]) - # print("------------- end") - - # create_test_interactive_user("Lucas2") - # print("report_definitions") \ No newline at end of file + test_data = load_json_data("../../test-data.json") + create_default_registry() + id_data = load_parameter_from_custom_parameters(file_path="testCustomParameters.json", parameter="ID") + for index, record in enumerate(test_data["registries"]["records"]): + insuree_id = id_data.get(f"ID_{index + 1}", record.pop("id")) + try: + insuree = create_test_insuree( + insuree_id=insuree_id, + last_name=record.pop('LastName'), + other_names=record.pop('FirstName'), + insuree_uuid=record.pop('uuid'), + json_ext=record + ) + except IntegrityError as e: + print(f"Error while creating insuree for ID {insuree_id}. Error: {str(e)}") diff --git a/examples/openIMIS/test_entrypoint.sh b/examples/openIMIS/test_entrypoint.sh index 3f48cbb..a81a09c 100755 --- a/examples/openIMIS/test_entrypoint.sh +++ b/examples/openIMIS/test_entrypoint.sh @@ -1,8 +1,12 @@ #!/bin/bash docker-compose build --no-cache db backend - docker-compose up -d db backend docker-compose exec backend chmod +x /check_service_availability.sh docker-compose exec backend bash /check_service_availability.sh + +docker-compose exec backend python /createTestCustomParameters.py +container_id=$(docker-compose ps -q backend) +docker cp $container_id:/openimis-be/openIMIS/testCustomParameters.json ./testCustomParameters.json + docker-compose exec backend python /import_data.py diff --git a/test/openAPI/features/data_create.feature b/test/openAPI/features/data_create.feature index 3b9ac3b..2f64b13 100644 --- a/test/openAPI/features/data_create.feature +++ b/test/openAPI/features/data_create.feature @@ -6,7 +6,7 @@ Feature: API endpoint that allows users to create a new record in the database. Given The user wants to create a new record in the database When User sends POST request with given Information-Mediator-Client header, body, "registryname" as registryname and "111" as versionnumber - And User provides body with parameters: "EE378627342345" as ID, "Anna" as Firstname, "Stock" as LastName, "RR-1234567999" BirthCertificateID + And User provides body with parameters: "${ID_0}" as ID, "${FirstName_0}" as Firstname, "${LastName_0}" as LastName, "${BirthCertificateID_0}" as BirthCertificateID Then User receives a response from the POST /data/{registryname}/{versionnumber}/create endpoint And The POST /data/{registryname}/{versionnumber}/create endpoint response should be returned in a timely manner 15000ms And The POST /data/{registryname}/{versionnumber}/create endpoint response should have status 200 @@ -18,7 +18,7 @@ Feature: API endpoint that allows users to create a new record in the database. Given The user wants to create a new record in the database When User sends POST request with given Information-Mediator-Client header, body, "registryname" as registryname and "111" as versionnumber - And User provides body with parameters: "" as ID, "" as Firstname, "" as LastName, "" BirthCertificateID + And User provides body with parameters: "" as ID, "" as Firstname, "" as LastName, "" as BirthCertificateID Then User receives a response from the POST /data/{registryname}/{versionnumber}/create endpoint And The POST /data/{registryname}/{versionnumber}/create endpoint response should be returned in a timely manner 15000ms And The POST /data/{registryname}/{versionnumber}/create endpoint response should have status 200 @@ -26,8 +26,8 @@ Feature: API endpoint that allows users to create a new record in the database. And The POST /data/{registryname}/{versionnumber}/create endpoint response should match json schema Examples: Valid data - | ID |Firstname | LastName | BirthCertificateID - | EE34534123 | Zofia | Don | RR-5465679229 - | EE38778473 | Elsa | Smith | RR-1234537999 - | RR87483474 | Nick | Evans | RR-7898797999 - | PP84848484 | Thomas | Johnson | RR-8534567781 + | ID |Firstname | LastName | BirthCertificateID + | ${ID_1} | ${FirstName_1} | ${LastName_1} | ${BirthCertificateID_1} + | ${ID_2} | ${FirstName_2} | ${LastName_2} | ${BirthCertificateID_2} + | ${ID_3} | ${FirstName_3} | ${LastName_3} | ${BirthCertificateID_3} + | ${ID_4} | ${FirstName_4} | ${LastName_4} | ${BirthCertificateID_4} diff --git a/test/openAPI/features/support/data_create.js b/test/openAPI/features/support/data_create.js index 1014502..2a6bfe4 100644 --- a/test/openAPI/features/support/data_create.js +++ b/test/openAPI/features/support/data_create.js @@ -9,12 +9,12 @@ const { defaultExpectedResponseTime, contentTypeHeader, dataCreateResponseSchema, + replaceKeyWithValueFromJson, } = require('./helpers/helpers'); chai.use(require('chai-json-schema')); let specDataCreate; - const baseUrl = localhost + dataCreateEndpoint; const endpointTag = { tags: `@endpoint=/${dataCreateEndpoint}` }; @@ -41,14 +41,15 @@ When( ); When( - 'User provides body with parameters: {string} as ID, {string} as Firstname, {string} as LastName, {string} BirthCertificateID', - (ID, Firstname, LastName, BirthCertificateID) => - specDataCreate.withBody({ - ID: ID, - Firstname: Firstname, - LastName: LastName, - BirthCertificateID: BirthCertificateID, - }) + 'User provides body with parameters: {string} as ID, {string} as Firstname, {string} as LastName, {string} as BirthCertificateID', + function (ID, Firstname, LastName, BirthCertificateID) { + return specDataCreate.withBody({ + ID: replaceKeyWithValueFromJson(ID), + Firstname: replaceKeyWithValueFromJson(Firstname), + LastName: replaceKeyWithValueFromJson(LastName), + BirthCertificateID: replaceKeyWithValueFromJson(BirthCertificateID), + }); + } ); Then( diff --git a/test/openAPI/features/support/helpers/helpers.js b/test/openAPI/features/support/helpers/helpers.js index 443a0a3..830fb89 100644 --- a/test/openAPI/features/support/helpers/helpers.js +++ b/test/openAPI/features/support/helpers/helpers.js @@ -369,4 +369,27 @@ module.exports = { }, required: ['ID', 'FirstName', 'LastName', 'BirthCertificateID'], }, + replaceKeyWithValueFromJson, + readJsonFile, }; + +const fs = require('fs'); +let jsonData = readJsonFile('testCustomParameters.json') || readJsonFile('testDefaultParameters.json'); + +function replaceKeyWithValueFromJson(key) { + let processedKey = key.replace('${', '').replace('}', ''); + if (!jsonData.hasOwnProperty(processedKey)) { + throw new Error(`Key ${processedKey} not found in jsonData`); + } + return jsonData[processedKey]; +} + +function readJsonFile(fileName) { + try { + let rawData = fs.readFileSync(fileName); + return JSON.parse(rawData); + } catch (error) { + console.log(`Failed to read ${fileName}, returning null.`); + return null; + } +} diff --git a/test/openAPI/gherkin_param_manager.sh b/test/openAPI/gherkin_param_manager.sh new file mode 100755 index 0000000..835ea1c --- /dev/null +++ b/test/openAPI/gherkin_param_manager.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Define paths to custom and default parameters files +CUSTOM_JSON_PATH="testCustomParameters.json" +DEFAULT_JSON_PATH="testDefaultParameters.json" +REPORT_PATH=$1 + +if [ -f $CUSTOM_JSON_PATH ] && [ -f $DEFAULT_JSON_PATH ]; then + # Both files exist, compare their keys + CUSTOM_KEYS=$(jq -r 'keys[]' $CUSTOM_JSON_PATH | sort) + DEFAULT_KEYS=$(jq -r 'keys[]' $DEFAULT_JSON_PATH | sort) + + # Check if keys in custom and default json file match + if [ "$CUSTOM_KEYS" != "$DEFAULT_KEYS" ]; then + echo "Keys in $CUSTOM_JSON_PATH and $DEFAULT_JSON_PATH do not match. Exiting." + exit 1 + fi + + # If keys match then use custom json file + PARAMETERS_PATH="$CUSTOM_JSON_PATH" +elif [ -f $DEFAULT_JSON_PATH ]; then + # Only default parameters file exists, use it for substitution + PARAMETERS_PATH="$DEFAULT_JSON_PATH" +else + echo "No parameters file found. Continuing without parameter substitution." + PARAMETERS_PATH="" +fi + +# Replcae ${key} with values from selected json file in generated report file +if [ ! -z $PARAMETERS_PATH ]; then + echo "Replacing placeholders in $REPORT_PATH using $PARAMETERS_PATH." + export TMPDIR=/tmp + while IFS="=" read -r key value + do + sed "s/\${$key}/$value/g" "$REPORT_PATH" > temp.txt && mv temp.txt "$REPORT_PATH" + done < <(jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' $PARAMETERS_PATH) +fi diff --git a/test/openAPI/test-data.json b/test/openAPI/test-data.json index a78179b..3c79243 100644 --- a/test/openAPI/test-data.json +++ b/test/openAPI/test-data.json @@ -134,7 +134,7 @@ ] }, { - "uuid": "80bcb084-fa17-11ed-be56-0242ac120002", + "uuid": "80bcb084-fa17-11ed-be56-0242ac120003", "id": "EE378129277266", "FirstName": "John", "LastName": "Bon", @@ -204,7 +204,7 @@ "PersonalData": [] }, { - "uuid": "4784f9c5-c5d7-40ce-9e19-88c1a1a8a8908", + "uuid": "4784f9c5-c5d7-40ce-9e19-88c1a1a8a890", "id": "EE65346467865", "FirstName": "Alina", "LastName": "Soft", @@ -235,7 +235,6 @@ "BirthCertificateID": "RR-1112223331", "PersonalData": [] }, - "//records below have to be deleted in the test - don't use it to update etc.", { "uuid": "80bcb804-fa17-11ed-be56-0242ac120002", "id": "EE378627348834", diff --git a/test/openAPI/testDefaultParameters.json b/test/openAPI/testDefaultParameters.json new file mode 100644 index 0000000..1580fed --- /dev/null +++ b/test/openAPI/testDefaultParameters.json @@ -0,0 +1,54 @@ +{ + "csrf_token": "D2222", + "jwt_token": "D2223", + "ID_0": "1999", + "ID_1": "2000", + "ID_2": "2001", + "ID_3": "2002", + "ID_4": "2003", + "ID_5": "2004", + "ID_6": "2005", + "ID_7": "2006", + "ID_8": "2007", + "ID_9": "2008", + "ID_10": "2009", + "ID_11": "2010", + "ID_12": "2011", + "ID_13": "2012", + "ID_14": "2013", + "ID_15": "2014", + "ID_16": "2015", + "ID_17": "2016", + "ID_18": "2017", + "ID_19": "2018", + "ID_20": "2019", + "ID_21": "2020", + "ID_22": "2021", + "ID_23": "2022", + "ID_24": "2023", + "ID_25": "2024", + "ID_26": "2025", + "ID_27": "2026", + "ID_28": "2027", + "ID_29": "2028", + "ID_30": "2029", + "ID_31": "2030", + "ID_32": "2031", + "ID_33": "2032", + "ID_34": "2033", + "FirstName_0": "Anna", + "FirstName_1": "Zofia", + "FirstName_2": "Elsa", + "FirstName_3": "Nick", + "FirstName_4": "Thomas", + "LastName_0": "Stock", + "LastName_1": "Don", + "LastName_2": "Smith", + "LastName_3": "Evans", + "LastName_4": "Johnson", + "BirthCertificateID_0": "RR-1234567999", + "BirthCertificateID_1": "RR-5465679229", + "BirthCertificateID_2": "RR-1234537999", + "BirthCertificateID_3": "RR-7898797999", + "BirthCertificateID_4": "RR-8534567781" +} \ No newline at end of file