diff --git a/Dockerfile b/Dockerfile index f9ac362a13..fff789042e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM nvidia/cuda:11.4.2-devel-ubuntu20.04 ARG VERSION=2.0.16 -COPY app/flip/prod_00/UCLH app/site-1 +COPY app/site-1/ app/site-1 RUN ls -la \ && apt update \ diff --git a/KCH.Dockerfile b/KCH.Dockerfile deleted file mode 100644 index b6fe1836ac..0000000000 --- a/KCH.Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM nvidia/cuda:11.4.2-devel-ubuntu20.04 - -ARG VERSION=2.0.16 - -COPY app/flip/prod_00/KCH app/site-1 - -RUN ls -la \ - && apt update \ - && yes | apt-get install wget \ - && yes | apt install python3.8 \ - && yes | apt install python3-pip \ - && yes | apt-get install python3-venv \ - && python3 -m venv nvflare-env \ - && . nvflare-env/bin/activate \ - && python3 -m pip install -U pip \ - && python3 -m pip install -U setuptools \ - && python3 -m pip install nvflare==${VERSION} \ - && python3 -m pip install torch torchvision \ - && python3 -m pip install numpy \ - && python3 -m pip install pandas \ - && echo 'NVFlare client installation complete' diff --git a/app-DO-NOT-TOUCH/server/log.txt b/app-DO-NOT-TOUCH/server/log.txt new file mode 100644 index 0000000000..092d9987ee --- /dev/null +++ b/app-DO-NOT-TOUCH/server/log.txt @@ -0,0 +1,118 @@ +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +E0802 16:31:18.569397500 258 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659454278.569381400","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-02 16:31:18,570 - FederatedServer - INFO - starting secure server at localhost:30021 +deployed FL server trainer. +2022-08-02 16:31:18,609 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-02 16:31:18,610 - root - INFO - Server started +E0803 12:18:49.716859200 124 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659525529.716837100","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-03 12:18:49,717 - FederatedServer - INFO - starting secure server at localhost:30021 +deployed FL server trainer. +2022-08-03 12:18:49,767 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-03 12:18:49,768 - root - INFO - Server started +E0803 12:20:34.918747900 191 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659525634.918719100","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-03 12:20:34,919 - FederatedServer - INFO - starting secure server at localhost:30021 +deployed FL server trainer. +2022-08-03 12:20:34,965 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-03 12:20:34,966 - root - INFO - Server started +E0803 12:21:49.590190900 242 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659525709.590168500","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-03 12:21:49,590 - FederatedServer - INFO - starting secure server at localhost:30021 +deployed FL server trainer. +2022-08-03 12:21:49,670 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-03 12:21:49,670 - root - INFO - Server started +E0803 12:23:07.273247700 293 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659525787.273220200","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-03 12:23:07,273 - FederatedServer - INFO - starting secure server at localhost:30021 +deployed FL server trainer. +2022-08-03 12:23:07,313 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-03 12:23:07,315 - root - INFO - Server started +/usr/bin/python3: Error while finding module specification for 'nvflare.private.fed.app.server.server_train' (ModuleNotFoundError: No module named 'nvflare') +/usr/bin/python3: Error while finding module specification for 'nvflare.private.fed.app.server.server_train' (ModuleNotFoundError: No module named 'nvflare') +/usr/bin/python3: Error while finding module specification for 'nvflare.private.fed.app.server.server_train' (ModuleNotFoundError: No module named 'nvflare') +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +E0803 12:30:21.394731500 487 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659526221.394708200","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-03 12:30:21,395 - FederatedServer - INFO - starting insecure server at localhost:30021 +deployed FL server trainer. +2022-08-03 12:30:21,433 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-03 12:30:21,433 - root - INFO - Server started +HELLOOOOO +E0803 12:33:04.830003500 547 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659526384.829973700","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-03 12:33:04,830 - FederatedServer - INFO - starting insecure server at localhost:30021 +deployed FL server trainer. +2022-08-03 12:33:04,861 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-03 12:33:04,862 - root - INFO - Server started +HELLOOOOO +E0803 12:34:01.837287400 593 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659526441.837259700","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-03 12:34:01,837 - FederatedServer - INFO - starting insecure server at localhost:30021 +2022-08-03 12:34:01,838 - FederatedServer - INFO - HELLO - I am run from the repo +deployed FL server trainer. +2022-08-03 12:34:01,909 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-03 12:34:01,909 - root - INFO - Server started +HELLOOOOO +E0803 12:36:13.386874200 676 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659526573.386840700","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-03 12:36:13,387 - FederatedServer - INFO - starting insecure server at localhost:30021 +2022-08-03 12:36:13,388 - FederatedServer - INFO - HELLO - I am run from the repo +deployed FL server trainer. +2022-08-03 12:36:13,449 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-03 12:36:13,450 - root - INFO - Server started diff --git a/app-DO-NOT-TOUCH/server/startup/fed_server.json b/app-DO-NOT-TOUCH/server/startup/fed_server.json new file mode 100644 index 0000000000..7a1e8ce285 --- /dev/null +++ b/app-DO-NOT-TOUCH/server/startup/fed_server.json @@ -0,0 +1,26 @@ +{ + "format_version": 1, + "servers": [ + { + "name": "example_project", + "service": { + "target": "localhost:30021", + "options": [ + ["grpc.max_send_message_length", 2147483647], + ["grpc.max_receive_message_length", 2147483647] + ] + }, + "admin_host": "localhost", + "admin_port": 8003, + "ssl_private_key": "server.key", + "ssl_cert": "server.crt", + "ssl_root_cert": "rootCA.pem", + "max_num_clients": 100, + "heart_beat_timeout": 600, + "num_server_workers": 4, + "compression": "Gzip", + "admin_storage": "transfer", + "min_num_clients": 1 + } + ] +} diff --git a/app/flip/prod_00/KCH/startup/log.config b/app-DO-NOT-TOUCH/server/startup/log.config similarity index 100% rename from app/flip/prod_00/KCH/startup/log.config rename to app-DO-NOT-TOUCH/server/startup/log.config diff --git a/app-DO-NOT-TOUCH/server/startup/rootCA.pem b/app-DO-NOT-TOUCH/server/startup/rootCA.pem new file mode 100644 index 0000000000..26c710989e --- /dev/null +++ b/app-DO-NOT-TOUCH/server/startup/rootCA.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC/DCCAeSgAwIBAgIUUfDzw0/Su90HKPiq/valB7oUUzowDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwOTA3MjBaFw0yMzA3MjgwOTA3 +MjBaMA8xDTALBgNVBAMMBGZsaXAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC2TpA+kuytw7334oUKTa7JMycsR04FnNGw0psDkDCexoTBIH6EqNrFR9FT +y5hJnLJDZLBcekKiCC3kQOTNOhcLVrT4f4lnGT+AQAqRD0D3xjTnQeeFGFQVaO5f +IpXvka9lSJ1Daz1dYxlvjHf31HcVB9IkP0g9DbN69r9uI5630OOw0rg3SeHBfFYk +Re5WhgQ40bdgk+qdV7TbgJzRNQPp8MYrnsJ+WzFVusmlgkP19co3r+D0kzsEPsBp +Q/A/nzu8m2AapAqeHgOIvNeud3ANrQcY9zkzgwvGwO+27dV+BaA4+XGSPrkcH0ha +eyDz0fPDALED7zN0LxyaHxPQJWUjAgMBAAGjUDBOMB0GA1UdDgQWBBRp3FlfLnTM +YGTefvTaIucwEXBnVTAfBgNVHSMEGDAWgBRp3FlfLnTMYGTefvTaIucwEXBnVTAM +BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCh4CCWy50gEOc7ZYVM0gLF +NR34yfw21Q26YY+bEiKzm2VsE519cW07XvaUw3MYAUGJ5c5+3ZKL8Hb1qylPH0Qp +JFZp2avfEUuaaxwq3RHuBpUU2lOYF5vsr3rAAIiih8ep8ARBFAtKhr9s1S+etQ7t +ZZHeDlxdHKOL7Fn1QoJwjLn/+aeSKq3IL2KVInyGB8i2XXJ3jXHz+oTeITfzEyvM +bBtV+iiWBLu+bYKiBjsJFOH8N/KBXDcjCbmlTL+GAqrrzXtkaxZEINprfhnPnGq8 +SFgX6S7hEiLmwUpKYYXRdf0DJqaNQcuVzr4w2q4Cgrzqd9AKtoieA1/tlTBTbH95 +-----END CERTIFICATE----- diff --git a/app-DO-NOT-TOUCH/server/startup/server.crt b/app-DO-NOT-TOUCH/server/startup/server.crt new file mode 100644 index 0000000000..9f6c494e50 --- /dev/null +++ b/app-DO-NOT-TOUCH/server/startup/server.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICrzCCAZegAwIBAgIUDH0JR5rKCtgK5uAg7uD57DJwLvAwDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwOTA3MjBaFw0yMzA3MjgwOTA3 +MjBaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOQL8/Bvr9XGrAAKKTGGgy9NhQ9AZZtDJyIqRTypf9uMiRqdWHKX +CCmZmuP0DgWNKj6zRGvHojs3n16dwFUIY0gBSt5kCapybik9w9pk4Kvb5muGs6SN +Q8GAU1ZFmWIbD7BlHuOX0m85x51E4isHsqWtfnMKiHtPB8R9PVb6cBMk6XjmEed5 +ozhnnuqwEi4bQ6Qr8XKx2uztyfivnVvmrrG3cLYMgfn3kklf3n2TgyJfmMxlAWEH +UmdvybwEvvm/PnnEewF0ulHaABjTJU8Arov0w+nLqTBgolY06kQ+EsZsQEuFxN89 +ku0y28wwji3WDZ+zQsVGDv6Qe4DmYSawNDUCAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEARUeNizUF+J64LvJyxYYZioY9AcQj6GkAoX2+w/94KC0DXlSm2JZXNFROFRQr +G/1aDUFLvJaSKCwFY3EqkH6KjlOKWperIRfN4bzx+xwcWkswlu7sXr6D07bG0/qa +e0/EEHTqDsQxykjYZi7Ltk6XgljnbbMk2njRH4NChO/4Vl6f1k7yumGIh+C7lFWO +ED5grI9wsekL8AVhHsgFanZajx+xffZH2AJDBzqfxCXcrNnMfg5Y5Wnfyl6Iryr+ +i7oKH+6lFCwpK3dIT5CLShCriZctVD27fmcR4eQI7nlsBiGpBzZgMK0F77lN352o +nAvQvqX84EWLeAk+nsYhaFQWgw== +-----END CERTIFICATE----- diff --git a/app-DO-NOT-TOUCH/server/startup/server.key b/app-DO-NOT-TOUCH/server/startup/server.key new file mode 100644 index 0000000000..da6d3dfff9 --- /dev/null +++ b/app-DO-NOT-TOUCH/server/startup/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA5Avz8G+v1casAAopMYaDL02FD0Blm0MnIipFPKl/24yJGp1Y +cpcIKZma4/QOBY0qPrNEa8eiOzefXp3AVQhjSAFK3mQJqnJuKT3D2mTgq9vma4az +pI1DwYBTVkWZYhsPsGUe45fSbznHnUTiKweypa1+cwqIe08HxH09VvpwEyTpeOYR +53mjOGee6rASLhtDpCvxcrHa7O3J+K+dW+ausbdwtgyB+feSSV/efZODIl+YzGUB +YQdSZ2/JvAS++b8+ecR7AXS6UdoAGNMlTwCui/TD6cupMGCiVjTqRD4SxmxAS4XE +3z2S7TLbzDCOLdYNn7NCxUYO/pB7gOZhJrA0NQIDAQABAoIBAAHbn984d5a8FDsO +yRFlwmsu1vcDT8POh2LB8DR3t3QWHxeRBHjbDxTVBCNvbpWLy1hhsaU10LHQR7r6 +4AnNC8KOWkXdsh0hc0RxjHHwDe1WYcURB0cddR4PMnPzSEcXmxUMwQ4jGNkRZk7/ +gQECeemGK9yPrVc/CpXkfoROsRd1wUFuhSXmKtVFv+IEFTpzuE9L4kj9b453oPtP +DV6viGrrHrghCvJ1FU4VcyuB4b6cMgnZ/7p/fqoSXvvY5vmUbqo/XcqHykh1Kh93 ++EwGEgPd4r9QtmMuauKvsot8PkOhG3p7DDK2m8SxF3A7kF19zC6GJLyAjDjJ2gzY +5eNNL4ECgYEA7YU8Xc7W/GyhPzzAMm53j3dM6Pfv3+Tae2jbrTsoAL13zORlA9Xj +G3kzO7JFq3ELpaKg/EEBxxpAWqwgtT3hsOVIlSZ4tpK6eQs94RSjzOpb06O5QAzs +u3KBfV4pzyFuuw6QPbsaoSrKI7Ys0Hc9FK2cYbEsjR4+w1gaZj6IXqECgYEA9coG +gTtz67D/Urf1SWOTa6QZoc67tZaUAKnUkk+uxpsRHToRKUrWeM+OhBjjUD+5ee7W +iWjjdOD4VUpPz2RgXEDRq7r3Jk8Iz3e5HVzeIBZQTz0HhpB6zlrvV8+M21fPkYSg +PfZUMZuDF0d+a4oGisVwgmk+VZrLSsNNyQLf0RUCgYBupqpHvWtULzszXZbgJ6xV +TquqwI6kTFYoNeJZInzxU1LzRdd1YPgrPVxLzDI+uyxE3q2GtrYgTi96zkNyi8CP +i1E0ArcrvtFWQJF0QQxKE0fyIYzUtPC4JpECmZJ6p0WRqSgzSe4O9QM0fdMEzvfK +oVKABG0P1O5CTNIHncwj4QKBgQDaRry6HetXX4uPkVDjCOxNxqEAwpCOthff8vKV +sO/87o8nTn8ZFZ4cEe91voh4pu73qzSZOS3uiS/gxqrT+kddBi9t/A8q1Tt0y9T4 +j5WwF0Dg7cj2Ju74vWcaq3ZjALuUdpy8CjtBBuNHVDWgGSawLKE7H229JkftsD4v +Cj9r9QKBgQDs4vjnO0cwLr9M13G0g4ioH+TheuHg2VL3K5oZ3LXiwT1akg1Qki1t +6bgoxM0n22qct/QqoMt02KFAQYWFUXXymwY31e3jeIGmHPnuqs4MSzp9V/pt7Jyx +Qx6JtuSnjCzete2tGQG70QpfzjjVlYT+NWBbl42Kvx3KWdRZr89KCg== +-----END RSA PRIVATE KEY----- diff --git a/app-DO-NOT-TOUCH/server/startup/signature.pkl b/app-DO-NOT-TOUCH/server/startup/signature.pkl new file mode 100644 index 0000000000..f702fe2ede Binary files /dev/null and b/app-DO-NOT-TOUCH/server/startup/signature.pkl differ diff --git a/app/flip/prod_00/KCH/startup/start.sh b/app-DO-NOT-TOUCH/server/startup/start.sh similarity index 52% rename from app/flip/prod_00/KCH/startup/start.sh rename to app-DO-NOT-TOUCH/server/startup/start.sh index 16a38383c7..01e0389890 100644 --- a/app/flip/prod_00/KCH/startup/start.sh +++ b/app-DO-NOT-TOUCH/server/startup/start.sh @@ -1,3 +1,5 @@ #!/usr/bin/env bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -$DIR/sub_start.sh & +echo "WORKSPACE set to $DIR/.." +mkdir -p $DIR/../transfer +$DIR/sub_start.sh $1 & diff --git a/app/flip/prod_00/localhost/startup/start.sh b/app-DO-NOT-TOUCH/server/startup/stop_fl.sh similarity index 50% rename from app/flip/prod_00/localhost/startup/start.sh rename to app-DO-NOT-TOUCH/server/startup/stop_fl.sh index 16a38383c7..7b22fd6932 100644 --- a/app/flip/prod_00/localhost/startup/start.sh +++ b/app-DO-NOT-TOUCH/server/startup/stop_fl.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -$DIR/sub_start.sh & +echo "Shutdown request created. FL system will shutdown soon." +touch $DIR/../shutdown.fl diff --git a/app-DO-NOT-TOUCH/server/startup/sub_start.sh b/app-DO-NOT-TOUCH/server/startup/sub_start.sh new file mode 100644 index 0000000000..16aa99301f --- /dev/null +++ b/app-DO-NOT-TOUCH/server/startup/sub_start.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +host=$1 + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "WORKSPACE set to $DIR/.." +mkdir -p $DIR/../transfer + +SECONDS=0 +lst=-400 +restart_count=0 +start_fl() { + if [[ $(( $SECONDS - $lst )) -lt 300 ]]; then + ((restart_count++)) + else + restart_count=0 + fi + if [[ $(($SECONDS - $lst )) -lt 300 && $restart_count -ge 5 ]]; then + echo "System is in trouble and unable to start the task!!!!!" + rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl + exit + fi + lst=$SECONDS +((python3 -u -m nvflare.private.fed.app.server.server_train -m $DIR/.. -s fed_server.json --set secure_train=false config_folder=config host=${host} 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | tee -a $DIR/../log.txt &) + pid=`cat $DIR/../pid.fl` +} + +stop_fl() { + if [[ ! -f "$DIR/../pid.fl" ]]; then + echo "No pid.fl. No need to kill process." + return + fi + pid=`cat $DIR/../pid.fl` + sleep 10 + kill -0 ${pid} 2> /dev/null 1>&2 + if [[ $? -ne 0 ]]; then + echo "Process alreday terminated" + return + fi + kill -9 $pid + rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl +} + +if [[ -f "$DIR/../daemon_pid.fl" ]]; then + dpid=`cat $DIR/../daemon_pid.fl` + kill -0 ${dpid} 2> /dev/null 1>&2 + if [[ $? -eq 0 ]]; then + echo "There seems to be one instance, pid=$dpid, running." + echo "If you are sure it's not the case, please kill process $dpid." + exit + fi + rm -f $DIR/../daemon_pid.fl +fi + +echo $BASHPID > $DIR/../daemon_pid.fl + +while true +do + sleep 5 + if [[ ! -f "$DIR/../pid.fl" ]]; then + start_fl + continue + fi + pid=`cat $DIR/../pid.fl` + kill -0 ${pid} 2> /dev/null 1>&2 + if [[ $? -ne 0 ]]; then + if [[ -f "$DIR/../shutdown.fl" ]]; then + echo "Gracefully shutdown." + break + fi + start_fl + continue + fi + if [[ -f "$DIR/../shutdown.fl" ]]; then + echo "About to shutdown." + stop_fl + break + fi + if [[ -f "$DIR/../restart.fl" ]]; then + echo "About to restart." + stop_fl + fi +done + +rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl diff --git a/app-DO-NOT-TOUCH/site-1/startup/client.crt b/app-DO-NOT-TOUCH/site-1/startup/client.crt new file mode 100644 index 0000000000..6e0207e178 --- /dev/null +++ b/app-DO-NOT-TOUCH/site-1/startup/client.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICqTCCAZGgAwIBAgIUf13NVfNWe8mM1ambruKbVckOFvYwDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwOTA3MjBaFw0yMzA3MjgwOTA3 +MjBaMA4xDDAKBgNVBAMMA0tDSDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAIQx0Pf9cYKMoH+wtc451gXNbPFdSgl+rpus2RKeqnIjo5aZIgD3C0++mtdi +wXvLRnEzbYT6nA8CzJ7pXbqv+0/ngG7FnaRqNhbfdHkif8gG8tvbRDKm02ZSOr4u +e9Nrdu2AN3JsT4fI1lHRSi6XrgBDlkPu8H1BTJlouZzrqqusdBvDbIF7mwVil1r1 +K/BEOr8U9ltljWRYTqyMQ3Gmf8F9p66dErypkHBo8OsEuv1YG/47fjws9D3XWEeR +aIAK/9op1sTGM6ypGstB3v2S03R6+2kWzPRUXLQ1mq/iA+RJtZDs+/HZLMeOVF81 +0PKg54Gsqxvre7QBCJPOd7//4qMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEArVMq +3avbxzlNEzD0kgIlmRIyrDU4r7PQw953aTwxrb1ks/Z8axiSyeeyBzrZMeceYXjY +KWL61gbMABM2O+EV9gHxULGf9aLGogeJ8PAX8mXpkoYeRsdyuu/JCDPAl9JtQFMc +qrCAxIxvCQq6MgV+Y88sF8WeyZym2WqSi2kaGiOQ7F71YN7tuxD+0zXwKarN14p6 +WSqNtbXPNyakLerq8EsbbWozEN+OMIJupTLYScpl56eiWh02mfz+G4qLqpgoGsJu +1c7pMn5aXHvjaGvi+9YypNFgnv4CkxXF4W1I9uF0asI2m1QVXS9zW+ii2L9RAyiS +ZZdH1ZK+OWuQezzmdw== +-----END CERTIFICATE----- diff --git a/app-DO-NOT-TOUCH/site-1/startup/client.key b/app-DO-NOT-TOUCH/site-1/startup/client.key new file mode 100644 index 0000000000..50f44d66c4 --- /dev/null +++ b/app-DO-NOT-TOUCH/site-1/startup/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAhDHQ9/1xgoygf7C1zjnWBc1s8V1KCX6um6zZEp6qciOjlpki +APcLT76a12LBe8tGcTNthPqcDwLMnulduq/7T+eAbsWdpGo2Ft90eSJ/yAby29tE +MqbTZlI6vi5702t27YA3cmxPh8jWUdFKLpeuAEOWQ+7wfUFMmWi5nOuqq6x0G8Ns +gXubBWKXWvUr8EQ6vxT2W2WNZFhOrIxDcaZ/wX2nrp0SvKmQcGjw6wS6/Vgb/jt+ +PCz0PddYR5FogAr/2inWxMYzrKkay0He/ZLTdHr7aRbM9FRctDWar+ID5Em1kOz7 +8dksx45UXzXQ8qDngayrG+t7tAEIk853v//iowIDAQABAoIBADB6nGn1lgUMUWfC +hEnmx+hE16kETzyAKsSIdOrxKQzGqR13dxLnhkxuWBaY7t/ZQjINbCKSTVzr5+J3 +LppiI0XAXkdBTXiYQtHMjMhHsJjIg5PBK5dvuqMRkN1h6qM0C985YBwkQF5Jdt92 +Y0AG/Ba4soijPaqRoee4kilL+8EM7HG6OMz2HBEx3cok2dMHiNK77ZDFUSFWsP/H +e6kKXh7sFpmnlfLXYC/hOOYU3oEaC/HrZ7X6aGCb5FzU5Oi7lQdhREq1geRpglbj +GRhSsheHaj+W9/4Klw0inpMqnbgo27p2jcTZXtlWI25uRMYRFbRMmhCSPbsbqYP2 +PPOeLTECgYEAtzJjGZTa9rXEe7V4ws6FsL1r9UYzDYqTiZDbnm9PYB8wlViUwoBZ +MghAam03N3btVcCD/wNDsGIish+3EzYbiwzf/Zc9bhV/FB4A7MM0tLmMNUG0Y9ZN +iHxkIRRd6eZsvQu8HTD8Qct+h2a+N050qf9m/PdZML20La0uMvdlP4cCgYEAuLqx +p1uAL6mPa203r+GANSc1qm94eziMWwjlP355N7FcM7MVKde/uw/tS1cIlMfrHV8Q +KiTijQCfBBhJxdDoLFF+ITmzVEIxN/Jd+4ssOUAPtgm3q5f7JKljjciqgG4lXC8E +QkRKkYZvveyrHqIhFHFllqd/VG15/WUKpZukcwUCgYALgtkDRH01v1onQ9Hgu8Ve +w6hC8zRGB40ZucZzAB+GtK8C01wVlKcGI8qYZWBw1cI3/y7MGVJphOEVPDTuuhmk +9Rwl/RjZD/S6OOBqACIRsQ8EUaqaE18v0lEgWv1Dszzl7BTX6LoREE5VB7mwE/f0 +rp20C37JN0JZFGawwSbVhQKBgFZhxu1RllGzYoODoQS32DIC0c+PLLHCMa/oKvtT +8u8eGtV6Kq1UXC3nNrmCS1r3WXR0naR3dxEkdzd8/BJqrAb9XJPA3aDLCyT0Swpz +fAyRbkdbp+2vthrgstL8QO6pkwLWYZFzScNfMCWJ5EMA4aMjm7OlQ5+BSgg6FY53 +h6DZAoGAcD+C9cuhVkwojQMkm1K3aGL2e032+lFMCo0dGwl3oiLLiVlaqt2Mj3Qo +sxv5Y9loWtzOrOr8SjFSVpGLoo3CG+HyvpTyi7/lp0rRjO9b+vIwMVJ3uraFLJ1W +iEep5vckChSdfTAD8Dx739Dcu7ePEzn5kvyQJ6LKY17v9URSF5M= +-----END RSA PRIVATE KEY----- diff --git a/app-DO-NOT-TOUCH/site-1/startup/fed_client.json b/app-DO-NOT-TOUCH/site-1/startup/fed_client.json new file mode 100644 index 0000000000..bc2a8c77c1 --- /dev/null +++ b/app-DO-NOT-TOUCH/site-1/startup/fed_client.json @@ -0,0 +1,22 @@ +{ + "format_version": 1, + "servers": [ + { + "name": "example_project", + "service": { + "target": "localhost:30021", + "options": [ + ["grpc.max_send_message_length", 2147483647], + ["grpc.max_receive_message_length", 2147483647] + ] + } + } + ], + "client": { + "retry_timeout": 30, + "ssl_private_key": "client.key", + "ssl_cert": "client.crt", + "ssl_root_cert": "rootCA.pem", + "compression": "Gzip" + } +} diff --git a/app/flip/prod_00/UCLH/startup/log.config b/app-DO-NOT-TOUCH/site-1/startup/log.config similarity index 100% rename from app/flip/prod_00/UCLH/startup/log.config rename to app-DO-NOT-TOUCH/site-1/startup/log.config diff --git a/app-DO-NOT-TOUCH/site-1/startup/rootCA.pem b/app-DO-NOT-TOUCH/site-1/startup/rootCA.pem new file mode 100644 index 0000000000..26c710989e --- /dev/null +++ b/app-DO-NOT-TOUCH/site-1/startup/rootCA.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC/DCCAeSgAwIBAgIUUfDzw0/Su90HKPiq/valB7oUUzowDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwOTA3MjBaFw0yMzA3MjgwOTA3 +MjBaMA8xDTALBgNVBAMMBGZsaXAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC2TpA+kuytw7334oUKTa7JMycsR04FnNGw0psDkDCexoTBIH6EqNrFR9FT +y5hJnLJDZLBcekKiCC3kQOTNOhcLVrT4f4lnGT+AQAqRD0D3xjTnQeeFGFQVaO5f +IpXvka9lSJ1Daz1dYxlvjHf31HcVB9IkP0g9DbN69r9uI5630OOw0rg3SeHBfFYk +Re5WhgQ40bdgk+qdV7TbgJzRNQPp8MYrnsJ+WzFVusmlgkP19co3r+D0kzsEPsBp +Q/A/nzu8m2AapAqeHgOIvNeud3ANrQcY9zkzgwvGwO+27dV+BaA4+XGSPrkcH0ha +eyDz0fPDALED7zN0LxyaHxPQJWUjAgMBAAGjUDBOMB0GA1UdDgQWBBRp3FlfLnTM +YGTefvTaIucwEXBnVTAfBgNVHSMEGDAWgBRp3FlfLnTMYGTefvTaIucwEXBnVTAM +BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCh4CCWy50gEOc7ZYVM0gLF +NR34yfw21Q26YY+bEiKzm2VsE519cW07XvaUw3MYAUGJ5c5+3ZKL8Hb1qylPH0Qp +JFZp2avfEUuaaxwq3RHuBpUU2lOYF5vsr3rAAIiih8ep8ARBFAtKhr9s1S+etQ7t +ZZHeDlxdHKOL7Fn1QoJwjLn/+aeSKq3IL2KVInyGB8i2XXJ3jXHz+oTeITfzEyvM +bBtV+iiWBLu+bYKiBjsJFOH8N/KBXDcjCbmlTL+GAqrrzXtkaxZEINprfhnPnGq8 +SFgX6S7hEiLmwUpKYYXRdf0DJqaNQcuVzr4w2q4Cgrzqd9AKtoieA1/tlTBTbH95 +-----END CERTIFICATE----- diff --git a/app-DO-NOT-TOUCH/site-1/startup/signature.pkl b/app-DO-NOT-TOUCH/site-1/startup/signature.pkl new file mode 100644 index 0000000000..96d596f828 Binary files /dev/null and b/app-DO-NOT-TOUCH/site-1/startup/signature.pkl differ diff --git a/app-DO-NOT-TOUCH/site-1/startup/start.sh b/app-DO-NOT-TOUCH/site-1/startup/start.sh new file mode 100644 index 0000000000..acde7ae582 --- /dev/null +++ b/app-DO-NOT-TOUCH/site-1/startup/start.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +site_name=site-NNN +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "WORKSPACE set to $DIR/.." +mkdir -p $DIR/../transfer +if [ $# -eq 0 ] ; then + echo "No sever name is provided. Default server (localhost) is used" + echo "Usage: start.sh >" + server=localhost +elif [ $# -eq 1 ] ; then + server=$1 +elif [ $# -eq 2 ] ; then + server=$1 + site_name=$2 +else + echo "Usage: start.sh >" + exit +fi +sleep 1 +$DIR/sub_start.sh $site_name $server & diff --git a/app-DO-NOT-TOUCH/site-1/startup/stop_fl.sh b/app-DO-NOT-TOUCH/site-1/startup/stop_fl.sh new file mode 100644 index 0000000000..7b22fd6932 --- /dev/null +++ b/app-DO-NOT-TOUCH/site-1/startup/stop_fl.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "Shutdown request created. FL system will shutdown soon." +touch $DIR/../shutdown.fl diff --git a/app/flip/prod_00/KCH/startup/sub_start.sh b/app-DO-NOT-TOUCH/site-1/startup/sub_start.sh similarity index 77% rename from app/flip/prod_00/KCH/startup/sub_start.sh rename to app-DO-NOT-TOUCH/site-1/startup/sub_start.sh index 69cf217f6c..1eefbbfe66 100644 --- a/app/flip/prod_00/KCH/startup/sub_start.sh +++ b/app-DO-NOT-TOUCH/site-1/startup/sub_start.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +client=$1 +server=$2 + + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "WORKSPACE set to $DIR/.." mkdir -p $DIR/../transfer @@ -20,9 +24,8 @@ start_fl() { exit fi lst=$SECONDS -((python3 -u -m nvflare.private.fed.app.client.client_train -m $DIR/.. -s fed_client.json --set secure_train=true uid=KCH config_folder=config 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | (tee -a $DIR/../log.txt & echo $! >&4 ) 4>$DIR/../tee.fl ) +((python3 -u -m nvflare.private.fed.app.client.client_train -m $DIR/.. -s fed_client.json --set secure_train=true uid=${client} config_folder=config host=${server} 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | tee -a $DIR/../log.txt &) pid=`cat $DIR/../pid.fl` - echo "new pid ${pid}" } stop_fl() { @@ -31,16 +34,14 @@ stop_fl() { return fi pid=`cat $DIR/../pid.fl` - sleep 5 + sleep 10 kill -0 ${pid} 2> /dev/null 1>&2 if [[ $? -ne 0 ]]; then - echo "Process already terminated" + echo "Process alreday terminated" return fi kill -9 $pid - pid=`cat $DIR/../tee.fl` - kill -9 $pid - rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl + rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl } if [[ -f "$DIR/../daemon_pid.fl" ]]; then @@ -48,7 +49,7 @@ if [[ -f "$DIR/../daemon_pid.fl" ]]; then kill -0 ${dpid} 2> /dev/null 1>&2 if [[ $? -eq 0 ]]; then echo "There seems to be one instance, pid=$dpid, running." - echo "If you are sure it's not the case, please kill process $dpid and then remove daemon_pid.fl in $DIR/.." + echo "If you are sure it's not the case, please kill process $dpid." exit fi rm -f $DIR/../daemon_pid.fl @@ -60,7 +61,6 @@ while true do sleep 5 if [[ ! -f "$DIR/../pid.fl" ]]; then - echo "start fl because of no pid.fl" start_fl continue fi @@ -71,7 +71,6 @@ do echo "Gracefully shutdown." break fi - echo "start fl because process of ${pid} does not exist" start_fl continue fi @@ -86,4 +85,4 @@ do fi done -rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl $DIR/../tee.fl +rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl diff --git a/app/flip/prod_00/KCH.zip b/app/flip/prod_00/KCH.zip deleted file mode 100644 index 522cb6cf50..0000000000 Binary files a/app/flip/prod_00/KCH.zip and /dev/null differ diff --git a/app/flip/prod_00/KCH/startup/client.crt b/app/flip/prod_00/KCH/startup/client.crt deleted file mode 100644 index 263d54d06d..0000000000 --- a/app/flip/prod_00/KCH/startup/client.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICqTCCAZGgAwIBAgIUQILxz9fT18ApJDFNKPXR0y1ew4EwDQYJKoZIhvcNAQEL -BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwODA4NDBaFw0yMzA3MjgwODA4 -NDBaMA4xDDAKBgNVBAMMA0tDSDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAMLc77/LudsOyQ348EUAQ6eaF3CSdZm/nxw2dHMvvQ9HwRPIARonS3YLiOyP -3m9N1hlCOySc1HkvVXq0C/QtcLf/J25nnYEHo+YYUWjgxeRRXQ/VecCkF0RJhMXM -cgsHBFDIlN4V9md+duh/N78mu5PXTSm/YJ0i3JXu6XGI4kvmNOsMtv4CpZgIN2Us -1V3TzKiUcwDKCRkPzejSdkX4ndemTE/MWxBX9vQ2f1Xm7S56Rtv8wVlV47UM3Tzu -qoDDw+38ELG2Gd/jgl4v2HBFS6yuLP8bTVVVIgU2/jN1xCnfOWiHcJS7kvWrFo/0 -WTj6TPUEYhFjitR8Bp/yWCS1uX0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfGe8 -8dnUSldOSlAX0s/m7rxOoaQYZZ1YTT7gY17V73p7PaSJi073bPY1YIm6iWGH3UG7 -um/kcMCaPIbX8qPbJS5Q80pz2qw0C2k1jNR/Qi1rb0njjHqtkW55gUo2CIMRY5qP -H8260c5o3B22ExxneprNLRyHGAW+u/oxjXhXX5ZMtiD4pVnEwpYlgUq131GmIknc -y+uGoDEUY+aspGUF2GT4MANzDxPMV37lj519pfs/LuqKxZ+ivtKW75640mbGYsqM -b+iqcX06zlUcBQuRQ1S5wThyL6grG+29tGkonn+LY/c+Xi0S80rK22p2xo9nlYCL -p5ug5ouigMuLJY80Hw== ------END CERTIFICATE----- diff --git a/app/flip/prod_00/KCH/startup/client.key b/app/flip/prod_00/KCH/startup/client.key deleted file mode 100644 index 8859226e3c..0000000000 --- a/app/flip/prod_00/KCH/startup/client.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAwtzvv8u52w7JDfjwRQBDp5oXcJJ1mb+fHDZ0cy+9D0fBE8gB -GidLdguI7I/eb03WGUI7JJzUeS9VerQL9C1wt/8nbmedgQej5hhRaODF5FFdD9V5 -wKQXREmExcxyCwcEUMiU3hX2Z3526H83vya7k9dNKb9gnSLcle7pcYjiS+Y06wy2 -/gKlmAg3ZSzVXdPMqJRzAMoJGQ/N6NJ2Rfid16ZMT8xbEFf29DZ/VebtLnpG2/zB -WVXjtQzdPO6qgMPD7fwQsbYZ3+OCXi/YcEVLrK4s/xtNVVUiBTb+M3XEKd85aIdw -lLuS9asWj/RZOPpM9QRiEWOK1HwGn/JYJLW5fQIDAQABAoIBABFGymDcXKmN2YiK -Gi9b8yOgd1SNP6z2ftJfB5sQz/fTWkx5LpPViQy5wBWNOeDvazyJETpRR0tJRZBf -9zFaeARwNFxHBoFinobvgARG7XStPA2D1UwdoEx3IjTxR+kvYZQR/JTmKJcLcNpz -MQubEQVuENLTO2sjJKCMcpkU1E97PUw0B4W51xyd7x1bUAczeJ2LwL8VQlX1rD8y -q1FCVjoGuKLLx1z/OP+rs83aqCu0TrtHRfaAEth81oCdQsqR14AeEuOr3qvL3y0I -Bg73faVZ1XvODNK04CSBDeD81KKaXUjsKZrSoqle27HrIWwafhy/0VCDZ3UH2J9Y -C1kXeGECgYEAyxoGEk5LETgM6cPtaWp0A0TjPIB618QQpPWnwNali8r1UDanR2ad -P4mwI3y5w/729V4NJiCngPSfXzAzO/lWreXvDgx5p/C/VjzaviPhOLNJSwzE9+Ty -vvUeEvGYyvuIV/Qd8/8yuPMcQur71idwdELdUuU8ObdMfNmYoYcvI20CgYEA9Z2Y -fOmjO1t2szF0DeLJSsP3E4a4NXxiZFLuU4pvfRhNj3M0F/fza2YAASbw2BpHIz/F -CRcmvMgm3hlBToGVIcCyAgSRCy7kPsVlccEZTWyhDSO5l1a2MTdjeHY9Nojx7SO4 -ykgGkoDZi+fQjujrqwp+Bvwvxv6YkO0GswTDFFECgYAuX77LJz1UrSulpKrsyvW6 -WVxZUCy8X3Q9dinbiic6iBkDO7duQivdeh7Ru4TfD7RryhHOht44P6K0JVlNdF0q -T7oJxydwBTRrCHIZwC+Itschb8T5pTkYCPfY83kjv9Md4guEztxSBpCJZwYhm9e9 -dBb8+oKJ2ySHzVdganGE8QKBgCMqQBXKQZOTAcigEPSX/qfUrMA/7onB2Dhy4ypU -zKeFXj1mihPQbSp4PRHtKFhYUs0KKUSm6uXmZiIwTXhW9yoQ86t0Sevm98jA7EhA -FEZQgjJmZgDz6jvbqUMq8pKnQrPoBGH5s+8vwXXVZ7e+Q1eGtgp7+r8r3l4JgYcJ -7IAxAoGABqSgYVCbqhStm2AIuptD8295pMqdGM7YhD7Gt+9FhMbNBR39u3F4YkZg -WheuFFn2JKdeeGNSJA3jk/9/naUjK1e3gWjYL/SxiB00mmCoiFi/pSEZE4J/ufJZ -KcnCbhu4FLpfqfEWmoax3p7FOQjCK+jZxkMXhLsS73DsdfqmUSs= ------END RSA PRIVATE KEY----- diff --git a/app/flip/prod_00/KCH/startup/client_context.tenseal b/app/flip/prod_00/KCH/startup/client_context.tenseal deleted file mode 100644 index 2a541b8fc9..0000000000 Binary files a/app/flip/prod_00/KCH/startup/client_context.tenseal and /dev/null differ diff --git a/app/flip/prod_00/KCH/startup/fed_client.json b/app/flip/prod_00/KCH/startup/fed_client.json deleted file mode 100644 index 03f91fe4b6..0000000000 --- a/app/flip/prod_00/KCH/startup/fed_client.json +++ /dev/null @@ -1 +0,0 @@ -{"format_version": 1, "enable_byoc": true, "servers": [{"name": "flip", "service": {"target": "api-private.flip.aicentre.co.uk:30021", "options": [["grpc.max_send_message_length", 2147483647], ["grpc.max_receive_message_length", 2147483647]]}}], "client": {"retry_timeout": 30, "ssl_private_key": "client.key", "ssl_cert": "client.crt", "ssl_root_cert": "rootCA.pem", "compression": "Gzip"}} \ No newline at end of file diff --git a/app/flip/prod_00/KCH/startup/readme.txt b/app/flip/prod_00/KCH/startup/readme.txt deleted file mode 100644 index 6daf7e9e16..0000000000 --- a/app/flip/prod_00/KCH/startup/readme.txt +++ /dev/null @@ -1,20 +0,0 @@ -********************************* -Federated Learning Client package -********************************* -The package includes at least the following files: -readme.txt -rootCA.pem -client.crt -client.key -fed_client.json -start.sh -sub_start.sh -stop_fl.sh - -Run start.sh to start the client. - -The rootCA.pem file is pointed by "ssl_root_cert" in fed_client.json. If you plan to move/copy it to a different place, -you will need to modify fed_client.json. The same applies to the other two files, client.crt and client.key. - -The client name in your submission to participate this Federated Learning project is embedded in the CN field of client -certificate, which uniquely identifies the participant. As such, please safeguard its private key, client.key. diff --git a/app/flip/prod_00/KCH/startup/rootCA.pem b/app/flip/prod_00/KCH/startup/rootCA.pem deleted file mode 100644 index b339aa5a5a..0000000000 --- a/app/flip/prod_00/KCH/startup/rootCA.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC/DCCAeSgAwIBAgIUFK+oYwHYf2xO5mzVwYdbq3E+jKUwDQYJKoZIhvcNAQEL -BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwODA4MzlaFw0yMzA3MjgwODA4 -MzlaMA8xDTALBgNVBAMMBGZsaXAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDLdULnMFceZbajeBXXphe4UMkybnPmZXFEL0xBc4ADJ32DLmXBOQhaBx1e -bY9zC6Z5j6ogNfs2Wb33WBPCnsov6xSTYSBU0VTVJBFwcP0iPzOj9K0CW6okzISu -iKBbUCPyx8fFHa+tMtzMBNUwD5gsp36jkH8diySHeezye+GqhWk0tJXEew1Zmmn7 -3mfG3jtsYm9tX1erX/Is6w+bxrLlgjLgvSzz/MXwdvrkYibAzkcLLk54DVzFFd1t -WoFwaUDIkJKSm9Hu+Pmcfw5Uv7Eq2keNyC2RkpPKvFD6vrrJRu//KDhEUT/n09N2 -2eB1nTuGd5Lx072+V2PZAG0d+d6JAgMBAAGjUDBOMB0GA1UdDgQWBBTjMZAjvHtJ -UFm0g8IeO1C+HU4WiDAfBgNVHSMEGDAWgBTjMZAjvHtJUFm0g8IeO1C+HU4WiDAM -BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBz0SQuSM+Az/BmrlIvioAz -Mj+7CzUe2aQdcg4NVtiS5UbWJWct6VGTp30v3ebtWDxdb/mWKGHfoK0nB6IfW4ya -hPFTmzC1b961uUOsKk+ZdnOteVtw4b1VxNolm61pWR863LPsNlBrRpzjDzKCU7Zd -0Aj0w2ILRPfuFjoeRDp0T9RHyVzuYtaqStpB7T83AZWZhHNnvhcLdnxuo6E+Snxn -CnQrzNrO2T1olAtXQXy/a7ZEUYlny8aSS3YEIleAecw3bmjKfXe5Pr8Pqi6bX3QE -/coJEuxWpRqbXBcwGduJHH/uvHHlqVX05Jvl1Qv+R16/s9qKB5YeTqnFdXGerIWy ------END CERTIFICATE----- diff --git a/app/flip/prod_00/KCH/startup/signature.pkl b/app/flip/prod_00/KCH/startup/signature.pkl deleted file mode 100644 index a5f8a7e050..0000000000 Binary files a/app/flip/prod_00/KCH/startup/signature.pkl and /dev/null differ diff --git a/app/flip/prod_00/KCH/startup/stop_fl.sh b/app/flip/prod_00/KCH/startup/stop_fl.sh deleted file mode 100644 index 82e72a2db6..0000000000 --- a/app/flip/prod_00/KCH/startup/stop_fl.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -echo "Shutdown request created. Wait for local FL process to shutdown." -touch $DIR/../shutdown.fl -sleep 5 -if [[ ! -f "$DIR/../pid.fl" ]]; then - echo "No pid.fl. No need to kill process." - exit -fi -pid=`cat $DIR/../pid.fl` -kill -0 ${pid} 2> /dev/null 1>&2 -if [[ $? -ne 0 ]]; then - echo "Process already terminated" - exit -fi -kill -9 $pid -pid=`cat $DIR/../tee.fl` -kill -9 $pid -rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl -echo "Shutdown process finished." diff --git a/app/flip/prod_00/UCLH.zip b/app/flip/prod_00/UCLH.zip deleted file mode 100644 index af11390741..0000000000 Binary files a/app/flip/prod_00/UCLH.zip and /dev/null differ diff --git a/app/flip/prod_00/UCLH/startup/client.crt b/app/flip/prod_00/UCLH/startup/client.crt deleted file mode 100644 index a00a82b3aa..0000000000 --- a/app/flip/prod_00/UCLH/startup/client.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICqjCCAZKgAwIBAgIUFvM3y+rStZ+QVk4/BsHpK+4HD5AwDQYJKoZIhvcNAQEL -BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwODA4MzlaFw0yMzA3MjgwODA4 -MzlaMA8xDTALBgNVBAMMBFVDTEgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQCtYhsCFzBf4KQIuPe18LL4gCSh9JKv72eSgt/MEvP4WveWCuk9scBGlyhu -+WLfiuyenVy6/jBjR6FBwr4MBctHhcA6YNCsnMz8gZp0Z+91zDpkOlSgMeajfYhq -c587WMQvyEkHVw1zgeVBZHW4CFuxJGPB8iybndXV2ieXUOSwUG4h7XDWTJNkrPJR -jp8LeJrwE6UuGVa2RVlon2zSbY/vRLEAu7o/xEpLWexgXuTulvi3qabJtahehFMc -1Gdpg22wun8z1n+rQk4h/r/Nwdqd/MblGmNarI71TBY8M/3yp8pHxMrtmlsl+ULN -oDPUfQLEHL7CM75uXfKJLb33Sd4NAgMBAAEwDQYJKoZIhvcNAQELBQADggEBACfZ -Wip5nQuJNSqlp5KGNNuEkmeZ9KQfnarngkFm186ga/dtK8k8e4gUXfWLGZZtN9GV -CrrhcYxZKhUguHIuUN2rNP844Gb7EZiCsSCh/7opSQwzmHeyH6w8NB087FwZ2MbG -6wt8dIBmLEy/vRoliOb7+sxf/pKJupy6l7316n1jiTfPxeAxqWXl6by1pCdjp2yz -UEVHVVrMge0PhSFPvdfROCOiNSAYH4gTb8SZR+uOyvI/Fg4aJldkVxC4exbSRoPm -ZPWBFLOUORAK3oibOSbsdfMqzPHoVnJCi6/9ghzyWi9FhI/gNU053vnlWat2xBt6 -jPpRmV+4NQSqqxp4Ayo= ------END CERTIFICATE----- diff --git a/app/flip/prod_00/UCLH/startup/client.key b/app/flip/prod_00/UCLH/startup/client.key deleted file mode 100644 index e8c704d6c7..0000000000 --- a/app/flip/prod_00/UCLH/startup/client.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEArWIbAhcwX+CkCLj3tfCy+IAkofSSr+9nkoLfzBLz+Fr3lgrp -PbHARpcobvli34rsnp1cuv4wY0ehQcK+DAXLR4XAOmDQrJzM/IGadGfvdcw6ZDpU -oDHmo32IanOfO1jEL8hJB1cNc4HlQWR1uAhbsSRjwfIsm53V1donl1DksFBuIe1w -1kyTZKzyUY6fC3ia8BOlLhlWtkVZaJ9s0m2P70SxALu6P8RKS1nsYF7k7pb4t6mm -ybWoXoRTHNRnaYNtsLp/M9Z/q0JOIf6/zcHanfzG5RpjWqyO9UwWPDP98qfKR8TK -7ZpbJflCzaAz1H0CxBy+wjO+bl3yiS2990neDQIDAQABAoIBADrYIYd3nEzl/nYR -ZMzkiLFLOGOZEJHShEnWziLCSff/06iiQl/UbIfotGnZjQOGcGEy0i6Kp5yCSon5 -kKue6ct36vJFOn1i6DVIvUWFouFNc4HhxCfsV7UkHUDXxfKL4WXAKXzqwB90Nf0Y -HdXcTFZWHd9MlHQqQNFILYivBTaLPpugN4NUPjjudoR1Qx9EUKBZkJgbQAEek1T8 -XFzvJ+lbXiwWH9uNueKGl4WM3ZhCk18rwu3n8AFOuJjZYf8AMv9M+Q+iWFx77nT+ -WL0lo9LxlgS4bDcN2AsaPtCcyS4vuJ+hxmTSZEdhu81vtTEHiT3MNe4zmDnJbTBN -0v5WyZcCgYEA01pKOw5iJP/lYSFEMgoIYqYuPH+AxDEvPrxhTCe40Ldk+q4ZleYY -+//HeT+2lboGp5K8fl1ppsdLn5ZeNCKKMQnyBUjRrcji6R1lJw20sKs/alIUTZsc -2Ix1lHfAnEHup28EJ1E56xFuNcXzxAyWDUByERm1bKql4yxPRCqgVycCgYEA0gJ4 -aREUJBv/Qx7hH95eoPufo5R0NmqnY56LTmhtqoM0wWsgv0KjP/l4JmRhcZmaiS/Y -7cZDLfCSp6QQeywzQxdP0uKeM4P62Sq7s9CafUO1xYOuqlGjpVdwIAfGPC/btQNj -/jxKyYvSWxpKQlgqVSApcnRPICMOYDJzXW6zgasCgYB9UjBpzCGN5t5QNbWGJkTK -mdcYsFHeA3wMKw9WZMdFmzkZG1q4c3OlGQlib9EH/wIrIur50tHuvoWs3zCEoWZn -pYoIcCFEdrV1nKR5/7qAd0pm9LxtN+hmzwJ5QnjHLqEgTsebuKIrLG65G+NrADuJ -KAl68k+NW7J5fbCSruPcSQKBgAd4NQYvqFK8eDDP+Vvj0ePR71SnJBcHZhrB+dx5 -FGV7MzYp9Ek5Oos8+UwMR/b9aqS0gfe7FIqewuiZikiHQu+A9cLfcpcMe7Cbt39m -zQflj6KN03mVW8e2u3M4M5HSdfMTQsw5IsPWaE37OmbvsdnDQhYjEulmtc0wHjBV -jqbvAoGBAL7nBgZbjehq8DRFyFfhG9Y3o37Vmm8r9ctPvPc5ojaxLKxDY57ep7bB -ebRJPkEo37RcHsaBbwb0zbq7US/cY00uG8fz1gwe2617tifeD8srPVhTmTmj18zp -+wpalEb8foNOHHEJRDgEbAiR5vbwcCtw5vWy76wzKEXAZmGkxwoj ------END RSA PRIVATE KEY----- diff --git a/app/flip/prod_00/UCLH/startup/client_context.tenseal b/app/flip/prod_00/UCLH/startup/client_context.tenseal deleted file mode 100644 index 2a541b8fc9..0000000000 Binary files a/app/flip/prod_00/UCLH/startup/client_context.tenseal and /dev/null differ diff --git a/app/flip/prod_00/UCLH/startup/fed_client.json b/app/flip/prod_00/UCLH/startup/fed_client.json deleted file mode 100644 index 03f91fe4b6..0000000000 --- a/app/flip/prod_00/UCLH/startup/fed_client.json +++ /dev/null @@ -1 +0,0 @@ -{"format_version": 1, "enable_byoc": true, "servers": [{"name": "flip", "service": {"target": "api-private.flip.aicentre.co.uk:30021", "options": [["grpc.max_send_message_length", 2147483647], ["grpc.max_receive_message_length", 2147483647]]}}], "client": {"retry_timeout": 30, "ssl_private_key": "client.key", "ssl_cert": "client.crt", "ssl_root_cert": "rootCA.pem", "compression": "Gzip"}} \ No newline at end of file diff --git a/app/flip/prod_00/UCLH/startup/readme.txt b/app/flip/prod_00/UCLH/startup/readme.txt deleted file mode 100644 index 6daf7e9e16..0000000000 --- a/app/flip/prod_00/UCLH/startup/readme.txt +++ /dev/null @@ -1,20 +0,0 @@ -********************************* -Federated Learning Client package -********************************* -The package includes at least the following files: -readme.txt -rootCA.pem -client.crt -client.key -fed_client.json -start.sh -sub_start.sh -stop_fl.sh - -Run start.sh to start the client. - -The rootCA.pem file is pointed by "ssl_root_cert" in fed_client.json. If you plan to move/copy it to a different place, -you will need to modify fed_client.json. The same applies to the other two files, client.crt and client.key. - -The client name in your submission to participate this Federated Learning project is embedded in the CN field of client -certificate, which uniquely identifies the participant. As such, please safeguard its private key, client.key. diff --git a/app/flip/prod_00/UCLH/startup/rootCA.pem b/app/flip/prod_00/UCLH/startup/rootCA.pem deleted file mode 100644 index b339aa5a5a..0000000000 --- a/app/flip/prod_00/UCLH/startup/rootCA.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC/DCCAeSgAwIBAgIUFK+oYwHYf2xO5mzVwYdbq3E+jKUwDQYJKoZIhvcNAQEL -BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwODA4MzlaFw0yMzA3MjgwODA4 -MzlaMA8xDTALBgNVBAMMBGZsaXAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDLdULnMFceZbajeBXXphe4UMkybnPmZXFEL0xBc4ADJ32DLmXBOQhaBx1e -bY9zC6Z5j6ogNfs2Wb33WBPCnsov6xSTYSBU0VTVJBFwcP0iPzOj9K0CW6okzISu -iKBbUCPyx8fFHa+tMtzMBNUwD5gsp36jkH8diySHeezye+GqhWk0tJXEew1Zmmn7 -3mfG3jtsYm9tX1erX/Is6w+bxrLlgjLgvSzz/MXwdvrkYibAzkcLLk54DVzFFd1t -WoFwaUDIkJKSm9Hu+Pmcfw5Uv7Eq2keNyC2RkpPKvFD6vrrJRu//KDhEUT/n09N2 -2eB1nTuGd5Lx072+V2PZAG0d+d6JAgMBAAGjUDBOMB0GA1UdDgQWBBTjMZAjvHtJ -UFm0g8IeO1C+HU4WiDAfBgNVHSMEGDAWgBTjMZAjvHtJUFm0g8IeO1C+HU4WiDAM -BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBz0SQuSM+Az/BmrlIvioAz -Mj+7CzUe2aQdcg4NVtiS5UbWJWct6VGTp30v3ebtWDxdb/mWKGHfoK0nB6IfW4ya -hPFTmzC1b961uUOsKk+ZdnOteVtw4b1VxNolm61pWR863LPsNlBrRpzjDzKCU7Zd -0Aj0w2ILRPfuFjoeRDp0T9RHyVzuYtaqStpB7T83AZWZhHNnvhcLdnxuo6E+Snxn -CnQrzNrO2T1olAtXQXy/a7ZEUYlny8aSS3YEIleAecw3bmjKfXe5Pr8Pqi6bX3QE -/coJEuxWpRqbXBcwGduJHH/uvHHlqVX05Jvl1Qv+R16/s9qKB5YeTqnFdXGerIWy ------END CERTIFICATE----- diff --git a/app/flip/prod_00/UCLH/startup/signature.pkl b/app/flip/prod_00/UCLH/startup/signature.pkl deleted file mode 100644 index 3bb4960a8f..0000000000 Binary files a/app/flip/prod_00/UCLH/startup/signature.pkl and /dev/null differ diff --git a/app/flip/prod_00/UCLH/startup/stop_fl.sh b/app/flip/prod_00/UCLH/startup/stop_fl.sh deleted file mode 100644 index 82e72a2db6..0000000000 --- a/app/flip/prod_00/UCLH/startup/stop_fl.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -echo "Shutdown request created. Wait for local FL process to shutdown." -touch $DIR/../shutdown.fl -sleep 5 -if [[ ! -f "$DIR/../pid.fl" ]]; then - echo "No pid.fl. No need to kill process." - exit -fi -pid=`cat $DIR/../pid.fl` -kill -0 ${pid} 2> /dev/null 1>&2 -if [[ $? -ne 0 ]]; then - echo "Process already terminated" - exit -fi -kill -9 $pid -pid=`cat $DIR/../tee.fl` -kill -9 $pid -rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl -echo "Shutdown process finished." diff --git a/app/flip/prod_00/localhost.zip b/app/flip/prod_00/localhost.zip deleted file mode 100644 index 6174277b09..0000000000 Binary files a/app/flip/prod_00/localhost.zip and /dev/null differ diff --git a/app/flip/prod_00/localhost/startup/fed_server.json b/app/flip/prod_00/localhost/startup/fed_server.json deleted file mode 100644 index d9f111d5f5..0000000000 --- a/app/flip/prod_00/localhost/startup/fed_server.json +++ /dev/null @@ -1 +0,0 @@ -{"format_version": 1, "enable_byoc": true, "servers": [{"name": "flip", "service": {"target": "localhost:8002", "options": [["grpc.max_send_message_length", 2147483647], ["grpc.max_receive_message_length", 2147483647]]}, "admin_host": "localhost", "admin_port": 8003, "admin_storage": "transfer", "ssl_private_key": "server.key", "ssl_cert": "server.crt", "ssl_root_cert": "rootCA.pem", "max_num_clients": 100, "heart_beat_timeout": 600, "num_server_workers": 4, "compression": "Gzip"}]} \ No newline at end of file diff --git a/app/flip/prod_00/localhost/startup/readme.txt b/app/flip/prod_00/localhost/startup/readme.txt deleted file mode 100644 index f17fc49514..0000000000 --- a/app/flip/prod_00/localhost/startup/readme.txt +++ /dev/null @@ -1,21 +0,0 @@ -********************************* -Federated Learning Server package -********************************* -The package includes at least the following files: -readme.txt -rootCA.pem -server.crt -server.key -authorization.json -fed_server.json -start.sh -sub_start.sh -stop_fl.sh -signature.pkl - -Run start.sh to start the server. - -The rootCA.pem file is pointed by "ssl_root_cert" in fed_server.json. If you plan to move/copy it to a different place, -you will need to modify fed_server.json. The same applies to the other two files, server.crt and server.key. - -Please always safeguard the server.key. diff --git a/app/flip/prod_00/localhost/startup/rootCA.pem b/app/flip/prod_00/localhost/startup/rootCA.pem deleted file mode 100644 index b339aa5a5a..0000000000 --- a/app/flip/prod_00/localhost/startup/rootCA.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC/DCCAeSgAwIBAgIUFK+oYwHYf2xO5mzVwYdbq3E+jKUwDQYJKoZIhvcNAQEL -BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwODA4MzlaFw0yMzA3MjgwODA4 -MzlaMA8xDTALBgNVBAMMBGZsaXAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDLdULnMFceZbajeBXXphe4UMkybnPmZXFEL0xBc4ADJ32DLmXBOQhaBx1e -bY9zC6Z5j6ogNfs2Wb33WBPCnsov6xSTYSBU0VTVJBFwcP0iPzOj9K0CW6okzISu -iKBbUCPyx8fFHa+tMtzMBNUwD5gsp36jkH8diySHeezye+GqhWk0tJXEew1Zmmn7 -3mfG3jtsYm9tX1erX/Is6w+bxrLlgjLgvSzz/MXwdvrkYibAzkcLLk54DVzFFd1t -WoFwaUDIkJKSm9Hu+Pmcfw5Uv7Eq2keNyC2RkpPKvFD6vrrJRu//KDhEUT/n09N2 -2eB1nTuGd5Lx072+V2PZAG0d+d6JAgMBAAGjUDBOMB0GA1UdDgQWBBTjMZAjvHtJ -UFm0g8IeO1C+HU4WiDAfBgNVHSMEGDAWgBTjMZAjvHtJUFm0g8IeO1C+HU4WiDAM -BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBz0SQuSM+Az/BmrlIvioAz -Mj+7CzUe2aQdcg4NVtiS5UbWJWct6VGTp30v3ebtWDxdb/mWKGHfoK0nB6IfW4ya -hPFTmzC1b961uUOsKk+ZdnOteVtw4b1VxNolm61pWR863LPsNlBrRpzjDzKCU7Zd -0Aj0w2ILRPfuFjoeRDp0T9RHyVzuYtaqStpB7T83AZWZhHNnvhcLdnxuo6E+Snxn -CnQrzNrO2T1olAtXQXy/a7ZEUYlny8aSS3YEIleAecw3bmjKfXe5Pr8Pqi6bX3QE -/coJEuxWpRqbXBcwGduJHH/uvHHlqVX05Jvl1Qv+R16/s9qKB5YeTqnFdXGerIWy ------END CERTIFICATE----- diff --git a/app/flip/prod_00/localhost/startup/server.crt b/app/flip/prod_00/localhost/startup/server.crt deleted file mode 100644 index a9ab9e7342..0000000000 --- a/app/flip/prod_00/localhost/startup/server.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICrzCCAZegAwIBAgIUX2u7+B/pOtvYqnZXEMhd9p9JvLMwDQYJKoZIhvcNAQEL -BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwODA4MzlaFw0yMzA3MjgwODA4 -MzlaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAPDgn0jN1Gken3yfFDMta4uMSigKJ0wrwBtvx4oI2QSIZ2za/a76 -UABF10XMbSlmcUWaGP3vRTToJKHGbhHyyCyyUfMP0gJGqCEPNKLq1lT3QuNUi78n -wRH00YKhauw1V2olFlhteMhT9ERNTSAD53hEs7ueD/xn+340qZir8cATme8hem7W -1Eg5SUsJ7hRnR0w3+c4OMtV6wstzi5Qn5fpagiQkOYAMDIst29TV/m1EhyqfGOED -JCOUM09JrwmiIXcqEgEM2qLRu2TSIljnTMLWHtHwMpxn8DSexMnKmrMVmvbiN5gf -+nLquNnRexVDbh9CAkYhkKXlxWomzEzRbpECAwEAATANBgkqhkiG9w0BAQsFAAOC -AQEAhGZyv3HJGxK2KzncX/TuzWWGD1jAM7ngUsuy0SQZ/wI4NKT+Yhpv5c5bdHDy -7rJL5yrbqalEr/dol5K1BjmC3XFtliBMOu7tQxQRr95UaqYUsBMbZ9/BU/rSHECK -UDCZ0xv5LAbmB6hcU7inY/tRm1uGYIEVj/dxSezh8p6bi9F5glGJwvotbIrH0gq4 -AKfCKmr8uBEdWqpMcz3AC8UQzYjKPf8HblRy3SnOozXqoWs4Bu8oa4XXFQlL08ka -rs+J4tv09Ee1v4v1mzFvRKRFiy5Gnve1YVcMBuM9BwVPupqzdv9LqyVp33+7hhgL -wzc0Omc+3nOouMubvc2VfRVnUA== ------END CERTIFICATE----- diff --git a/app/flip/prod_00/localhost/startup/server.key b/app/flip/prod_00/localhost/startup/server.key deleted file mode 100644 index 7634026d3b..0000000000 --- a/app/flip/prod_00/localhost/startup/server.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA8OCfSM3UaR6ffJ8UMy1ri4xKKAonTCvAG2/HigjZBIhnbNr9 -rvpQAEXXRcxtKWZxRZoY/e9FNOgkocZuEfLILLJR8w/SAkaoIQ80ourWVPdC41SL -vyfBEfTRgqFq7DVXaiUWWG14yFP0RE1NIAPneESzu54P/Gf7fjSpmKvxwBOZ7yF6 -btbUSDlJSwnuFGdHTDf5zg4y1XrCy3OLlCfl+lqCJCQ5gAwMiy3b1NX+bUSHKp8Y -4QMkI5QzT0mvCaIhdyoSAQzaotG7ZNIiWOdMwtYe0fAynGfwNJ7EycqasxWa9uI3 -mB/6cuq42dF7FUNuH0ICRiGQpeXFaibMTNFukQIDAQABAoIBAA6CfV3RlkK5x0Im -w/z8Nq76haoJSIo4VXNhpZutQkNZYx+FxoPAjMqjal8IPCgjBV47tNcxD1JrTG+Z -SB6ZKRE2MaUKVixwbKbHJ3TL4BrQpS0T9fIPFzx5aphrmEj/wpMAhmrPt1eb5Qor -POLZkVsniksaL5zU96uHyRKLAv4+ohmGXp2PHGA/MYK07tE7REsLyNbkjFYY6OkF -7hwjB8tHrwdOJUtqtpYb4fE5PZvm9SjNjOg8aO1DD1o2xXgx1HDfGL69H5CgXvv/ -QSocGQjz7Z94MaoSyRIG8CfCkGH6gNFgBLtqu4NnnXJTOVqTEvouMV0IR9JIUeQ1 -TKJvYuUCgYEA8oMw8sjTuvCLVBuW+ScUyqhmuuabG42KNy7sJJKEYFPpkt+a0/0C -UWhjhxHoATpoi5WZbpP4LE+GuXs7Q5oNuSDCeInp0Fhc/krXOMpuNUVwx6kUtpkL -/wegH/f1Pbh/ASskcDDLnLzC2LZTSB1BGYy32Qva8Cvyt/jrdTZpnb0CgYEA/kYm -5SFSAYEUcrQMewlMW3NKXwpgg3wwvlOBWMSkGA4A2+EQ8+2dGILQjZfPibqtWozK -BlUxaW3jDvmcaBkzQgQcf797eM501CnFZyyRwbXIY7KrU6kxS7sKqT+z3EVZH2vP -ikHNNo0djuvslvGPvgNxkKC9ulzKoBNvMhO6r2UCgYA3JKmeqxep7ERiLZvYr5BE -vKIPYZxnWXnSLnM3pbFADhSQdxPHnEJIrInlDqdIkDVxd+TZhf/qY3Lo/iJtrrvs -Gozdx85It679i5sAqNcCefW5ELNLCeX2aY95dXJoRGlspJi2ggHqc+mCSYMKZMyG -uL6dNdrkvbmFJ66yilrdqQKBgQCDaiBKBZmLe3nk+lcE875J+WDzk7pj2jia9S+B -gEfKwtoAyeZoG2M4usssksxJQ3i6ts7mzPr3oTpbj/gc+QE72kIcaAtZWWKaEGic -tTUzp07E8B+J4oS+vhF5fZnoetLf4Wlm7DEyQ2aGajphnMEMlF0KQXktKy3x43lx -qS6v3QKBgCOT5WH7JhnAkvX3zDcmqcQdBYkytXDneNGdtbb1M6+3WtyRbo8MZ2un -80hnc9WAC5MLsQzFzaJKc7fZ9yIPclxMSNR4wBWoESQpWHwA7V0ZvfGCZJBossuR -yVD4Hewas5y2pWKesOqalYrcazAI9pZDGYVJ4yRjcIEDqMfKhXUd ------END RSA PRIVATE KEY----- diff --git a/app/flip/prod_00/localhost/startup/server_context.tenseal b/app/flip/prod_00/localhost/startup/server_context.tenseal deleted file mode 100644 index fc077fd082..0000000000 Binary files a/app/flip/prod_00/localhost/startup/server_context.tenseal and /dev/null differ diff --git a/app/flip/prod_00/localhost/startup/signature.pkl b/app/flip/prod_00/localhost/startup/signature.pkl deleted file mode 100644 index 0573ccd195..0000000000 Binary files a/app/flip/prod_00/localhost/startup/signature.pkl and /dev/null differ diff --git a/app/flip/prod_00/localhost/startup/stop_fl.sh b/app/flip/prod_00/localhost/startup/stop_fl.sh deleted file mode 100644 index 82e72a2db6..0000000000 --- a/app/flip/prod_00/localhost/startup/stop_fl.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -echo "Shutdown request created. Wait for local FL process to shutdown." -touch $DIR/../shutdown.fl -sleep 5 -if [[ ! -f "$DIR/../pid.fl" ]]; then - echo "No pid.fl. No need to kill process." - exit -fi -pid=`cat $DIR/../pid.fl` -kill -0 ${pid} 2> /dev/null 1>&2 -if [[ $? -ne 0 ]]; then - echo "Process already terminated" - exit -fi -kill -9 $pid -pid=`cat $DIR/../tee.fl` -kill -9 $pid -rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl -echo "Shutdown process finished." diff --git a/app/flip/state/cert.pkl b/app/flip/state/cert.pkl deleted file mode 100644 index 4d05b54b62..0000000000 Binary files a/app/flip/state/cert.pkl and /dev/null differ diff --git a/app/server/log.txt b/app/server/log.txt new file mode 100644 index 0000000000..0f5a641744 --- /dev/null +++ b/app/server/log.txt @@ -0,0 +1,55 @@ +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +The following files are not secure content. +fed_server.json +server.crt +server.key +rootCA.pem +E0802 16:31:18.569397500 258 socket_utils_common_posix.cc:223] check for SO_REUSEPORT: {"created":"@1659454278.569381400","description":"Protocol not available","errno":92,"file":"src/core/lib/iomgr/socket_utils_common_posix.cc","file_line":202,"os_error":"Protocol not available","syscall":"getsockopt(SO_REUSEPORT)"} +2022-08-02 16:31:18,570 - FederatedServer - INFO - starting secure server at localhost:30021 +deployed FL server trainer. +2022-08-02 16:31:18,609 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003 +2022-08-02 16:31:18,610 - root - INFO - Server started diff --git a/app/server/startup/fed_server.json b/app/server/startup/fed_server.json new file mode 100644 index 0000000000..7a1e8ce285 --- /dev/null +++ b/app/server/startup/fed_server.json @@ -0,0 +1,26 @@ +{ + "format_version": 1, + "servers": [ + { + "name": "example_project", + "service": { + "target": "localhost:30021", + "options": [ + ["grpc.max_send_message_length", 2147483647], + ["grpc.max_receive_message_length", 2147483647] + ] + }, + "admin_host": "localhost", + "admin_port": 8003, + "ssl_private_key": "server.key", + "ssl_cert": "server.crt", + "ssl_root_cert": "rootCA.pem", + "max_num_clients": 100, + "heart_beat_timeout": 600, + "num_server_workers": 4, + "compression": "Gzip", + "admin_storage": "transfer", + "min_num_clients": 1 + } + ] +} diff --git a/app/flip/prod_00/localhost/startup/log.config b/app/server/startup/log.config similarity index 100% rename from app/flip/prod_00/localhost/startup/log.config rename to app/server/startup/log.config diff --git a/app/server/startup/rootCA.pem b/app/server/startup/rootCA.pem new file mode 100644 index 0000000000..26c710989e --- /dev/null +++ b/app/server/startup/rootCA.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC/DCCAeSgAwIBAgIUUfDzw0/Su90HKPiq/valB7oUUzowDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwOTA3MjBaFw0yMzA3MjgwOTA3 +MjBaMA8xDTALBgNVBAMMBGZsaXAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC2TpA+kuytw7334oUKTa7JMycsR04FnNGw0psDkDCexoTBIH6EqNrFR9FT +y5hJnLJDZLBcekKiCC3kQOTNOhcLVrT4f4lnGT+AQAqRD0D3xjTnQeeFGFQVaO5f +IpXvka9lSJ1Daz1dYxlvjHf31HcVB9IkP0g9DbN69r9uI5630OOw0rg3SeHBfFYk +Re5WhgQ40bdgk+qdV7TbgJzRNQPp8MYrnsJ+WzFVusmlgkP19co3r+D0kzsEPsBp +Q/A/nzu8m2AapAqeHgOIvNeud3ANrQcY9zkzgwvGwO+27dV+BaA4+XGSPrkcH0ha +eyDz0fPDALED7zN0LxyaHxPQJWUjAgMBAAGjUDBOMB0GA1UdDgQWBBRp3FlfLnTM +YGTefvTaIucwEXBnVTAfBgNVHSMEGDAWgBRp3FlfLnTMYGTefvTaIucwEXBnVTAM +BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCh4CCWy50gEOc7ZYVM0gLF +NR34yfw21Q26YY+bEiKzm2VsE519cW07XvaUw3MYAUGJ5c5+3ZKL8Hb1qylPH0Qp +JFZp2avfEUuaaxwq3RHuBpUU2lOYF5vsr3rAAIiih8ep8ARBFAtKhr9s1S+etQ7t +ZZHeDlxdHKOL7Fn1QoJwjLn/+aeSKq3IL2KVInyGB8i2XXJ3jXHz+oTeITfzEyvM +bBtV+iiWBLu+bYKiBjsJFOH8N/KBXDcjCbmlTL+GAqrrzXtkaxZEINprfhnPnGq8 +SFgX6S7hEiLmwUpKYYXRdf0DJqaNQcuVzr4w2q4Cgrzqd9AKtoieA1/tlTBTbH95 +-----END CERTIFICATE----- diff --git a/app/server/startup/server.crt b/app/server/startup/server.crt new file mode 100644 index 0000000000..9f6c494e50 --- /dev/null +++ b/app/server/startup/server.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICrzCCAZegAwIBAgIUDH0JR5rKCtgK5uAg7uD57DJwLvAwDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwOTA3MjBaFw0yMzA3MjgwOTA3 +MjBaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOQL8/Bvr9XGrAAKKTGGgy9NhQ9AZZtDJyIqRTypf9uMiRqdWHKX +CCmZmuP0DgWNKj6zRGvHojs3n16dwFUIY0gBSt5kCapybik9w9pk4Kvb5muGs6SN +Q8GAU1ZFmWIbD7BlHuOX0m85x51E4isHsqWtfnMKiHtPB8R9PVb6cBMk6XjmEed5 +ozhnnuqwEi4bQ6Qr8XKx2uztyfivnVvmrrG3cLYMgfn3kklf3n2TgyJfmMxlAWEH +UmdvybwEvvm/PnnEewF0ulHaABjTJU8Arov0w+nLqTBgolY06kQ+EsZsQEuFxN89 +ku0y28wwji3WDZ+zQsVGDv6Qe4DmYSawNDUCAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEARUeNizUF+J64LvJyxYYZioY9AcQj6GkAoX2+w/94KC0DXlSm2JZXNFROFRQr +G/1aDUFLvJaSKCwFY3EqkH6KjlOKWperIRfN4bzx+xwcWkswlu7sXr6D07bG0/qa +e0/EEHTqDsQxykjYZi7Ltk6XgljnbbMk2njRH4NChO/4Vl6f1k7yumGIh+C7lFWO +ED5grI9wsekL8AVhHsgFanZajx+xffZH2AJDBzqfxCXcrNnMfg5Y5Wnfyl6Iryr+ +i7oKH+6lFCwpK3dIT5CLShCriZctVD27fmcR4eQI7nlsBiGpBzZgMK0F77lN352o +nAvQvqX84EWLeAk+nsYhaFQWgw== +-----END CERTIFICATE----- diff --git a/app/server/startup/server.key b/app/server/startup/server.key new file mode 100644 index 0000000000..da6d3dfff9 --- /dev/null +++ b/app/server/startup/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA5Avz8G+v1casAAopMYaDL02FD0Blm0MnIipFPKl/24yJGp1Y +cpcIKZma4/QOBY0qPrNEa8eiOzefXp3AVQhjSAFK3mQJqnJuKT3D2mTgq9vma4az +pI1DwYBTVkWZYhsPsGUe45fSbznHnUTiKweypa1+cwqIe08HxH09VvpwEyTpeOYR +53mjOGee6rASLhtDpCvxcrHa7O3J+K+dW+ausbdwtgyB+feSSV/efZODIl+YzGUB +YQdSZ2/JvAS++b8+ecR7AXS6UdoAGNMlTwCui/TD6cupMGCiVjTqRD4SxmxAS4XE +3z2S7TLbzDCOLdYNn7NCxUYO/pB7gOZhJrA0NQIDAQABAoIBAAHbn984d5a8FDsO +yRFlwmsu1vcDT8POh2LB8DR3t3QWHxeRBHjbDxTVBCNvbpWLy1hhsaU10LHQR7r6 +4AnNC8KOWkXdsh0hc0RxjHHwDe1WYcURB0cddR4PMnPzSEcXmxUMwQ4jGNkRZk7/ +gQECeemGK9yPrVc/CpXkfoROsRd1wUFuhSXmKtVFv+IEFTpzuE9L4kj9b453oPtP +DV6viGrrHrghCvJ1FU4VcyuB4b6cMgnZ/7p/fqoSXvvY5vmUbqo/XcqHykh1Kh93 ++EwGEgPd4r9QtmMuauKvsot8PkOhG3p7DDK2m8SxF3A7kF19zC6GJLyAjDjJ2gzY +5eNNL4ECgYEA7YU8Xc7W/GyhPzzAMm53j3dM6Pfv3+Tae2jbrTsoAL13zORlA9Xj +G3kzO7JFq3ELpaKg/EEBxxpAWqwgtT3hsOVIlSZ4tpK6eQs94RSjzOpb06O5QAzs +u3KBfV4pzyFuuw6QPbsaoSrKI7Ys0Hc9FK2cYbEsjR4+w1gaZj6IXqECgYEA9coG +gTtz67D/Urf1SWOTa6QZoc67tZaUAKnUkk+uxpsRHToRKUrWeM+OhBjjUD+5ee7W +iWjjdOD4VUpPz2RgXEDRq7r3Jk8Iz3e5HVzeIBZQTz0HhpB6zlrvV8+M21fPkYSg +PfZUMZuDF0d+a4oGisVwgmk+VZrLSsNNyQLf0RUCgYBupqpHvWtULzszXZbgJ6xV +TquqwI6kTFYoNeJZInzxU1LzRdd1YPgrPVxLzDI+uyxE3q2GtrYgTi96zkNyi8CP +i1E0ArcrvtFWQJF0QQxKE0fyIYzUtPC4JpECmZJ6p0WRqSgzSe4O9QM0fdMEzvfK +oVKABG0P1O5CTNIHncwj4QKBgQDaRry6HetXX4uPkVDjCOxNxqEAwpCOthff8vKV +sO/87o8nTn8ZFZ4cEe91voh4pu73qzSZOS3uiS/gxqrT+kddBi9t/A8q1Tt0y9T4 +j5WwF0Dg7cj2Ju74vWcaq3ZjALuUdpy8CjtBBuNHVDWgGSawLKE7H229JkftsD4v +Cj9r9QKBgQDs4vjnO0cwLr9M13G0g4ioH+TheuHg2VL3K5oZ3LXiwT1akg1Qki1t +6bgoxM0n22qct/QqoMt02KFAQYWFUXXymwY31e3jeIGmHPnuqs4MSzp9V/pt7Jyx +Qx6JtuSnjCzete2tGQG70QpfzjjVlYT+NWBbl42Kvx3KWdRZr89KCg== +-----END RSA PRIVATE KEY----- diff --git a/app/server/startup/signature.pkl b/app/server/startup/signature.pkl new file mode 100644 index 0000000000..f702fe2ede Binary files /dev/null and b/app/server/startup/signature.pkl differ diff --git a/app/flip/prod_00/UCLH/startup/start.sh b/app/server/startup/start.sh similarity index 52% rename from app/flip/prod_00/UCLH/startup/start.sh rename to app/server/startup/start.sh index 16a38383c7..01e0389890 100644 --- a/app/flip/prod_00/UCLH/startup/start.sh +++ b/app/server/startup/start.sh @@ -1,3 +1,5 @@ #!/usr/bin/env bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -$DIR/sub_start.sh & +echo "WORKSPACE set to $DIR/.." +mkdir -p $DIR/../transfer +$DIR/sub_start.sh $1 & diff --git a/app/server/startup/stop_fl.sh b/app/server/startup/stop_fl.sh new file mode 100644 index 0000000000..7b22fd6932 --- /dev/null +++ b/app/server/startup/stop_fl.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "Shutdown request created. FL system will shutdown soon." +touch $DIR/../shutdown.fl diff --git a/app/flip/prod_00/localhost/startup/sub_start.sh b/app/server/startup/sub_start.sh similarity index 79% rename from app/flip/prod_00/localhost/startup/sub_start.sh rename to app/server/startup/sub_start.sh index 3ef2a7ec09..2435413790 100644 --- a/app/flip/prod_00/localhost/startup/sub_start.sh +++ b/app/server/startup/sub_start.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +host=$1 + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "WORKSPACE set to $DIR/.." mkdir -p $DIR/../transfer @@ -18,9 +20,8 @@ start_fl() { exit fi lst=$SECONDS -((python3 -u -m nvflare.private.fed.app.server.server_train -m $DIR/.. -s fed_server.json --set secure_train=true config_folder=config 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | (tee -a $DIR/../log.txt & echo $! >&4 ) 4>$DIR/../tee.fl ) +((python3 -u -m nvflare.private.fed.app.server.server_train -m $DIR/.. -s fed_server.json --set secure_train=true config_folder=config host=${host} 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | tee -a $DIR/../log.txt &) pid=`cat $DIR/../pid.fl` - echo "new pid ${pid}" } stop_fl() { @@ -29,16 +30,14 @@ stop_fl() { return fi pid=`cat $DIR/../pid.fl` - sleep 5 + sleep 10 kill -0 ${pid} 2> /dev/null 1>&2 if [[ $? -ne 0 ]]; then - echo "Process already terminated" + echo "Process alreday terminated" return fi kill -9 $pid - pid=`cat $DIR/../tee.fl` - kill -9 $pid - rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl + rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl } if [[ -f "$DIR/../daemon_pid.fl" ]]; then @@ -46,7 +45,7 @@ if [[ -f "$DIR/../daemon_pid.fl" ]]; then kill -0 ${dpid} 2> /dev/null 1>&2 if [[ $? -eq 0 ]]; then echo "There seems to be one instance, pid=$dpid, running." - echo "If you are sure it's not the case, please kill process $dpid and then remove daemon_pid.fl in $DIR/.." + echo "If you are sure it's not the case, please kill process $dpid." exit fi rm -f $DIR/../daemon_pid.fl @@ -58,7 +57,6 @@ while true do sleep 5 if [[ ! -f "$DIR/../pid.fl" ]]; then - echo "start fl because of no pid.fl" start_fl continue fi @@ -69,7 +67,6 @@ do echo "Gracefully shutdown." break fi - echo "start fl because process of ${pid} does not exist" start_fl continue fi @@ -84,4 +81,4 @@ do fi done -rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl $DIR/../tee.fl +rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl diff --git a/app/site-1/startup/client.crt b/app/site-1/startup/client.crt new file mode 100644 index 0000000000..6e0207e178 --- /dev/null +++ b/app/site-1/startup/client.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICqTCCAZGgAwIBAgIUf13NVfNWe8mM1ambruKbVckOFvYwDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwOTA3MjBaFw0yMzA3MjgwOTA3 +MjBaMA4xDDAKBgNVBAMMA0tDSDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAIQx0Pf9cYKMoH+wtc451gXNbPFdSgl+rpus2RKeqnIjo5aZIgD3C0++mtdi +wXvLRnEzbYT6nA8CzJ7pXbqv+0/ngG7FnaRqNhbfdHkif8gG8tvbRDKm02ZSOr4u +e9Nrdu2AN3JsT4fI1lHRSi6XrgBDlkPu8H1BTJlouZzrqqusdBvDbIF7mwVil1r1 +K/BEOr8U9ltljWRYTqyMQ3Gmf8F9p66dErypkHBo8OsEuv1YG/47fjws9D3XWEeR +aIAK/9op1sTGM6ypGstB3v2S03R6+2kWzPRUXLQ1mq/iA+RJtZDs+/HZLMeOVF81 +0PKg54Gsqxvre7QBCJPOd7//4qMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEArVMq +3avbxzlNEzD0kgIlmRIyrDU4r7PQw953aTwxrb1ks/Z8axiSyeeyBzrZMeceYXjY +KWL61gbMABM2O+EV9gHxULGf9aLGogeJ8PAX8mXpkoYeRsdyuu/JCDPAl9JtQFMc +qrCAxIxvCQq6MgV+Y88sF8WeyZym2WqSi2kaGiOQ7F71YN7tuxD+0zXwKarN14p6 +WSqNtbXPNyakLerq8EsbbWozEN+OMIJupTLYScpl56eiWh02mfz+G4qLqpgoGsJu +1c7pMn5aXHvjaGvi+9YypNFgnv4CkxXF4W1I9uF0asI2m1QVXS9zW+ii2L9RAyiS +ZZdH1ZK+OWuQezzmdw== +-----END CERTIFICATE----- diff --git a/app/site-1/startup/client.key b/app/site-1/startup/client.key new file mode 100644 index 0000000000..50f44d66c4 --- /dev/null +++ b/app/site-1/startup/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAhDHQ9/1xgoygf7C1zjnWBc1s8V1KCX6um6zZEp6qciOjlpki +APcLT76a12LBe8tGcTNthPqcDwLMnulduq/7T+eAbsWdpGo2Ft90eSJ/yAby29tE +MqbTZlI6vi5702t27YA3cmxPh8jWUdFKLpeuAEOWQ+7wfUFMmWi5nOuqq6x0G8Ns +gXubBWKXWvUr8EQ6vxT2W2WNZFhOrIxDcaZ/wX2nrp0SvKmQcGjw6wS6/Vgb/jt+ +PCz0PddYR5FogAr/2inWxMYzrKkay0He/ZLTdHr7aRbM9FRctDWar+ID5Em1kOz7 +8dksx45UXzXQ8qDngayrG+t7tAEIk853v//iowIDAQABAoIBADB6nGn1lgUMUWfC +hEnmx+hE16kETzyAKsSIdOrxKQzGqR13dxLnhkxuWBaY7t/ZQjINbCKSTVzr5+J3 +LppiI0XAXkdBTXiYQtHMjMhHsJjIg5PBK5dvuqMRkN1h6qM0C985YBwkQF5Jdt92 +Y0AG/Ba4soijPaqRoee4kilL+8EM7HG6OMz2HBEx3cok2dMHiNK77ZDFUSFWsP/H +e6kKXh7sFpmnlfLXYC/hOOYU3oEaC/HrZ7X6aGCb5FzU5Oi7lQdhREq1geRpglbj +GRhSsheHaj+W9/4Klw0inpMqnbgo27p2jcTZXtlWI25uRMYRFbRMmhCSPbsbqYP2 +PPOeLTECgYEAtzJjGZTa9rXEe7V4ws6FsL1r9UYzDYqTiZDbnm9PYB8wlViUwoBZ +MghAam03N3btVcCD/wNDsGIish+3EzYbiwzf/Zc9bhV/FB4A7MM0tLmMNUG0Y9ZN +iHxkIRRd6eZsvQu8HTD8Qct+h2a+N050qf9m/PdZML20La0uMvdlP4cCgYEAuLqx +p1uAL6mPa203r+GANSc1qm94eziMWwjlP355N7FcM7MVKde/uw/tS1cIlMfrHV8Q +KiTijQCfBBhJxdDoLFF+ITmzVEIxN/Jd+4ssOUAPtgm3q5f7JKljjciqgG4lXC8E +QkRKkYZvveyrHqIhFHFllqd/VG15/WUKpZukcwUCgYALgtkDRH01v1onQ9Hgu8Ve +w6hC8zRGB40ZucZzAB+GtK8C01wVlKcGI8qYZWBw1cI3/y7MGVJphOEVPDTuuhmk +9Rwl/RjZD/S6OOBqACIRsQ8EUaqaE18v0lEgWv1Dszzl7BTX6LoREE5VB7mwE/f0 +rp20C37JN0JZFGawwSbVhQKBgFZhxu1RllGzYoODoQS32DIC0c+PLLHCMa/oKvtT +8u8eGtV6Kq1UXC3nNrmCS1r3WXR0naR3dxEkdzd8/BJqrAb9XJPA3aDLCyT0Swpz +fAyRbkdbp+2vthrgstL8QO6pkwLWYZFzScNfMCWJ5EMA4aMjm7OlQ5+BSgg6FY53 +h6DZAoGAcD+C9cuhVkwojQMkm1K3aGL2e032+lFMCo0dGwl3oiLLiVlaqt2Mj3Qo +sxv5Y9loWtzOrOr8SjFSVpGLoo3CG+HyvpTyi7/lp0rRjO9b+vIwMVJ3uraFLJ1W +iEep5vckChSdfTAD8Dx739Dcu7ePEzn5kvyQJ6LKY17v9URSF5M= +-----END RSA PRIVATE KEY----- diff --git a/app/site-1/startup/fed_client.json b/app/site-1/startup/fed_client.json new file mode 100644 index 0000000000..bc2a8c77c1 --- /dev/null +++ b/app/site-1/startup/fed_client.json @@ -0,0 +1,22 @@ +{ + "format_version": 1, + "servers": [ + { + "name": "example_project", + "service": { + "target": "localhost:30021", + "options": [ + ["grpc.max_send_message_length", 2147483647], + ["grpc.max_receive_message_length", 2147483647] + ] + } + } + ], + "client": { + "retry_timeout": 30, + "ssl_private_key": "client.key", + "ssl_cert": "client.crt", + "ssl_root_cert": "rootCA.pem", + "compression": "Gzip" + } +} diff --git a/app/site-1/startup/log.config b/app/site-1/startup/log.config new file mode 100644 index 0000000000..6b9761569b --- /dev/null +++ b/app/site-1/startup/log.config @@ -0,0 +1,27 @@ +[loggers] +keys=root,modelLogger + +[handlers] +keys=consoleHandler + +[formatters] +keys=fullFormatter + +[logger_root] +level=INFO +handlers=consoleHandler + +[logger_modelLogger] +level=DEBUG +handlers=consoleHandler +qualname=modelLogger +propagate=0 + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=fullFormatter +args=(sys.stdout,) + +[formatter_fullFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/app/site-1/startup/rootCA.pem b/app/site-1/startup/rootCA.pem new file mode 100644 index 0000000000..26c710989e --- /dev/null +++ b/app/site-1/startup/rootCA.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC/DCCAeSgAwIBAgIUUfDzw0/Su90HKPiq/valB7oUUzowDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAwwEZmxpcDAeFw0yMjA4MDIwOTA3MjBaFw0yMzA3MjgwOTA3 +MjBaMA8xDTALBgNVBAMMBGZsaXAwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC2TpA+kuytw7334oUKTa7JMycsR04FnNGw0psDkDCexoTBIH6EqNrFR9FT +y5hJnLJDZLBcekKiCC3kQOTNOhcLVrT4f4lnGT+AQAqRD0D3xjTnQeeFGFQVaO5f +IpXvka9lSJ1Daz1dYxlvjHf31HcVB9IkP0g9DbN69r9uI5630OOw0rg3SeHBfFYk +Re5WhgQ40bdgk+qdV7TbgJzRNQPp8MYrnsJ+WzFVusmlgkP19co3r+D0kzsEPsBp +Q/A/nzu8m2AapAqeHgOIvNeud3ANrQcY9zkzgwvGwO+27dV+BaA4+XGSPrkcH0ha +eyDz0fPDALED7zN0LxyaHxPQJWUjAgMBAAGjUDBOMB0GA1UdDgQWBBRp3FlfLnTM +YGTefvTaIucwEXBnVTAfBgNVHSMEGDAWgBRp3FlfLnTMYGTefvTaIucwEXBnVTAM +BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCh4CCWy50gEOc7ZYVM0gLF +NR34yfw21Q26YY+bEiKzm2VsE519cW07XvaUw3MYAUGJ5c5+3ZKL8Hb1qylPH0Qp +JFZp2avfEUuaaxwq3RHuBpUU2lOYF5vsr3rAAIiih8ep8ARBFAtKhr9s1S+etQ7t +ZZHeDlxdHKOL7Fn1QoJwjLn/+aeSKq3IL2KVInyGB8i2XXJ3jXHz+oTeITfzEyvM +bBtV+iiWBLu+bYKiBjsJFOH8N/KBXDcjCbmlTL+GAqrrzXtkaxZEINprfhnPnGq8 +SFgX6S7hEiLmwUpKYYXRdf0DJqaNQcuVzr4w2q4Cgrzqd9AKtoieA1/tlTBTbH95 +-----END CERTIFICATE----- diff --git a/app/site-1/startup/signature.pkl b/app/site-1/startup/signature.pkl new file mode 100644 index 0000000000..96d596f828 Binary files /dev/null and b/app/site-1/startup/signature.pkl differ diff --git a/app/site-1/startup/start.sh b/app/site-1/startup/start.sh new file mode 100644 index 0000000000..acde7ae582 --- /dev/null +++ b/app/site-1/startup/start.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +site_name=site-NNN +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "WORKSPACE set to $DIR/.." +mkdir -p $DIR/../transfer +if [ $# -eq 0 ] ; then + echo "No sever name is provided. Default server (localhost) is used" + echo "Usage: start.sh >" + server=localhost +elif [ $# -eq 1 ] ; then + server=$1 +elif [ $# -eq 2 ] ; then + server=$1 + site_name=$2 +else + echo "Usage: start.sh >" + exit +fi +sleep 1 +$DIR/sub_start.sh $site_name $server & diff --git a/app/site-1/startup/stop_fl.sh b/app/site-1/startup/stop_fl.sh new file mode 100644 index 0000000000..7b22fd6932 --- /dev/null +++ b/app/site-1/startup/stop_fl.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "Shutdown request created. FL system will shutdown soon." +touch $DIR/../shutdown.fl diff --git a/app/flip/prod_00/UCLH/startup/sub_start.sh b/app/site-1/startup/sub_start.sh similarity index 77% rename from app/flip/prod_00/UCLH/startup/sub_start.sh rename to app/site-1/startup/sub_start.sh index 2b87393762..1eefbbfe66 100644 --- a/app/flip/prod_00/UCLH/startup/sub_start.sh +++ b/app/site-1/startup/sub_start.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +client=$1 +server=$2 + + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "WORKSPACE set to $DIR/.." mkdir -p $DIR/../transfer @@ -20,9 +24,8 @@ start_fl() { exit fi lst=$SECONDS -((python3 -u -m nvflare.private.fed.app.client.client_train -m $DIR/.. -s fed_client.json --set secure_train=true uid=UCLH config_folder=config 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | (tee -a $DIR/../log.txt & echo $! >&4 ) 4>$DIR/../tee.fl ) +((python3 -u -m nvflare.private.fed.app.client.client_train -m $DIR/.. -s fed_client.json --set secure_train=true uid=${client} config_folder=config host=${server} 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | tee -a $DIR/../log.txt &) pid=`cat $DIR/../pid.fl` - echo "new pid ${pid}" } stop_fl() { @@ -31,16 +34,14 @@ stop_fl() { return fi pid=`cat $DIR/../pid.fl` - sleep 5 + sleep 10 kill -0 ${pid} 2> /dev/null 1>&2 if [[ $? -ne 0 ]]; then - echo "Process already terminated" + echo "Process alreday terminated" return fi kill -9 $pid - pid=`cat $DIR/../tee.fl` - kill -9 $pid - rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl + rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl } if [[ -f "$DIR/../daemon_pid.fl" ]]; then @@ -48,7 +49,7 @@ if [[ -f "$DIR/../daemon_pid.fl" ]]; then kill -0 ${dpid} 2> /dev/null 1>&2 if [[ $? -eq 0 ]]; then echo "There seems to be one instance, pid=$dpid, running." - echo "If you are sure it's not the case, please kill process $dpid and then remove daemon_pid.fl in $DIR/.." + echo "If you are sure it's not the case, please kill process $dpid." exit fi rm -f $DIR/../daemon_pid.fl @@ -60,7 +61,6 @@ while true do sleep 5 if [[ ! -f "$DIR/../pid.fl" ]]; then - echo "start fl because of no pid.fl" start_fl continue fi @@ -71,7 +71,6 @@ do echo "Gracefully shutdown." break fi - echo "start fl because process of ${pid} does not exist" start_fl continue fi @@ -86,4 +85,4 @@ do fi done -rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl $DIR/../tee.fl +rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl diff --git a/build_utils/build.py b/build_utils/build.py new file mode 100644 index 0000000000..dbba5f2e7e --- /dev/null +++ b/build_utils/build.py @@ -0,0 +1,47 @@ +import argparse +import subprocess +import yaml +import os + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-p", "--project_file", type=str, default="project.yml", help="file to describe FL project") + parser.add_argument("-w", "--workspace", type=str, default="workspace", help="directory used by provision") + parser.add_argument("--client_host", type=str, default="localhost", help="") + parser.add_argument("--client_port", type=int, default=8002, help="port number appened to client host") + parser.add_argument("--server_host", type=str, default="0.0.0.0", help="") + parser.add_argument("--server_port", type=int, default=8002, help="port number appened to server host") + + args = parser.parse_args() + + current_dir = os.getcwd() + + project_config_file_path = os.path.join(current_dir, args.project_file) + + if (os.path.isfile(project_config_file_path) == False): + raise FileNotFoundError(project_config_file_path) + + project_config = yaml.safe_load(open(project_config_file_path, "r")) + + static_file_args = project_config["builders"][2]["args"] + + static_file_args = { + "config_folder": "config", + "client_port": args.client_port, + "client_host": args.client_host, + "server_port": args.server_port, + "server_host": args.server_host, + "admin_port": 8003, + "admin_host": "localhost" + } + + project_config["builders"][2]["args"] = static_file_args + + yaml.safe_dump(project_config, open(project_config_file_path, "w")) + + subprocess.call(["python3", "-m", "nvflare.lighter.provision", "-p", args.project_file, "-w", args.workspace]) + + nvflare_dir = os.path.join(current_dir, "nvflare") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/build_utils/cert.py b/build_utils/cert.py new file mode 100644 index 0000000000..aa9f5e68e4 --- /dev/null +++ b/build_utils/cert.py @@ -0,0 +1,158 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import pickle + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID + +from nvflare.lighter.spec import Builder + + +def serialize_pri_key(pri_key): + return pri_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + + +def serialize_cert(cert): + return cert.public_bytes(serialization.Encoding.PEM) + + +class CertBuilder(Builder): + def __init__(self): + """Build certificate chain for every participant. + + Handles building (creating and self-signing) the root CA certificates, creating server, client and + admin certificates, and having them signed by the root CA for secure communication. If the state folder has + information about previously generated certs, it loads them back and reuses them. + """ + self.root_cert = None + self.persistent_state = dict() + + def initialize(self, ctx): + state_dir = self.get_state_dir(ctx) + cert_file = os.path.join(state_dir, "cert.pkl") + if os.path.exists(cert_file): + self.persistent_state = pickle.load(open(cert_file, "rb")) + self.serialized_cert = self.persistent_state["root_cert"] + self.root_cert = x509.load_pem_x509_certificate(self.serialized_cert, default_backend()) + self.pri_key = serialization.load_pem_private_key( + self.persistent_state["root_pri_key"], password=None, backend=default_backend() + ) + self.pub_key = self.pri_key.public_key() + self.subject = self.root_cert.subject + + def _build_root(self, subject): + if not self.persistent_state: + pri_key, pub_key = self._generate_keys() + self.subject = self._x509_name(subject) + issuer = self.subject + self.root_cert = self._generate_cert(self.subject, issuer, pri_key, pub_key, ca=True) + self.pri_key = pri_key + self.pub_key = pub_key + self.serialized_cert = serialize_cert(self.root_cert) + self.persistent_state["root_cert"] = self.serialized_cert + self.persistent_state["root_pri_key"] = serialize_pri_key(self.pri_key) + + def _build_write_cert_pair(self, participant, base_name, ctx): + subject = participant.subject + if self.persistent_state and subject in self.persistent_state: + cert = x509.load_pem_x509_certificate(self.persistent_state[subject]["cert"], default_backend()) + pri_key = serialization.load_pem_private_key( + self.persistent_state[subject]["pri_key"], password=None, backend=default_backend() + ) + else: + pri_key, cert = self.get_pri_key_cert(participant) + self.persistent_state[subject] = dict(cert=serialize_cert(cert), pri_key=serialize_pri_key(pri_key)) + dest_dir = self.get_kit_dir(participant, ctx) + with open(os.path.join(dest_dir, f"{base_name}.crt"), "wb") as f: + f.write(serialize_cert(cert)) + with open(os.path.join(dest_dir, f"{base_name}.key"), "wb") as f: + f.write(serialize_pri_key(pri_key)) + with open(os.path.join(dest_dir, "rootCA.pem"), "wb") as f: + f.write(self.serialized_cert) + + def build(self, study, ctx): + self._build_root(study.name) + ctx["root_cert"] = self.root_cert + ctx["root_pri_key"] = self.pri_key + server = study.get_participants_by_type("server") + self._build_write_cert_pair(server, "server", ctx) + + for client in study.get_participants_by_type("client", first_only=False): + self._build_write_cert_pair(client, "client", ctx) + + for admin in study.get_participants_by_type("admin", first_only=False): + self._build_write_cert_pair(admin, "client", ctx) + + def get_pri_key_cert(self, participant): + pri_key, pub_key = self._generate_keys() + subject = self._x509_name(participant.subject) + issuer = self.subject + cert = self._generate_cert(subject, issuer, self.pri_key, pub_key) + return pri_key, cert + + def _generate_keys(self): + pri_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) + pub_key = pri_key.public_key() + return pri_key, pub_key + + def _generate_cert(self, subject, issuer, signing_pri_key, subject_pub_key, valid_days=360, ca=False): + builder = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(subject_pub_key) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after( + # Our certificate will be valid for 360 days + datetime.datetime.utcnow() + + datetime.timedelta(days=valid_days) + # Sign our certificate with our private key + ) + ) + if ca: + builder = ( + builder.add_extension( + x509.SubjectKeyIdentifier.from_public_key(subject_pub_key), + critical=False, + ) + .add_extension( + x509.AuthorityKeyIdentifier.from_issuer_public_key(subject_pub_key), + critical=False, + ) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False) + ) + return builder.sign(signing_pri_key, hashes.SHA256(), default_backend()) + + def _x509_name(self, cn_name, org_name=None): + name = [x509.NameAttribute(NameOID.COMMON_NAME, cn_name)] + if org_name is not None: + name.append(x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)) + return x509.Name(name) + + def finalize(self, ctx): + return + # state_dir = self.get_state_dir(ctx) + # cert_file = os.path.join(state_dir, "cert.pkl") + # pickle.dump(self.persistent_state, open(cert_file, "wb")) diff --git a/app/flip/resources/flip_template.yml b/build_utils/flip_template.yml similarity index 100% rename from app/flip/resources/flip_template.yml rename to build_utils/flip_template.yml diff --git a/build_utils/generate_certs.py b/build_utils/generate_certs.py new file mode 100644 index 0000000000..7b7be78594 --- /dev/null +++ b/build_utils/generate_certs.py @@ -0,0 +1,105 @@ +import os +import pickle +import datetime + +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography import x509 +from cryptography.x509 import Certificate +from cryptography.x509.oid import NameOID + +def _generate_keys(): + pri_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) + pub_key = pri_key.public_key() + return pri_key, pub_key + + +def _generate_cert(subject, issuer, signing_pri_key, subject_pub_key, valid_days=360, ca=False): + builder = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(subject_pub_key) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after( + # Our certificate will be valid for 360 days + datetime.datetime.utcnow() + + datetime.timedelta(days=valid_days) + # Sign our certificate with our private key + ) + ) + if ca: + builder = ( + builder.add_extension( + x509.SubjectKeyIdentifier.from_public_key(subject_pub_key), + critical=False, + ) + .add_extension( + x509.AuthorityKeyIdentifier.from_issuer_public_key(subject_pub_key), + critical=False, + ) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False) + ) + return builder.sign(signing_pri_key, hashes.SHA256(), default_backend()) + + +def _x509_name(cn_name, org_name=None): + name = [x509.NameAttribute(NameOID.COMMON_NAME, cn_name)] + if org_name is not None: + name.append(x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)) + return x509.Name(name) + + +def serialize_pri_key(pri_key): + return pri_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + + +def serialize_cert(cert): + return cert.public_bytes(serialization.Encoding.PEM) + + +def _build_root(subject, issuer): + root_pri_key, root_pub_key = _generate_keys() + subject_name = _x509_name(subject) + issuer_name = _x509_name(issuer) + root_cert = _generate_cert(subject_name, issuer_name, root_pri_key, root_pub_key, ca=True) + # serialized_root_cert = serialize_cert(root_cert) + # serialized_pri_key = serialize_pri_key(pri_key) + return root_pri_key, root_pub_key, root_cert + + +def _build_cert_pair(subject, root_cert: Certificate, root_pri_key): + pri_key, pub_key = _generate_keys() + subject_name = _x509_name(subject) + issuer_name = root_cert.issuer + cert = _generate_cert(subject_name, issuer_name, root_pri_key, pub_key, ca=False) + return pri_key, cert + +working_dir = os.getcwd() +dest_dir = os.path.join(working_dir, "build-utils", "generated-certs") + +root_pri_key,root_pub_key,root_cert = _build_root("flip", "flip") + +server_pri_key, server_cert = _build_cert_pair("flip-server", root_cert, root_pri_key) + +client_pri_key, client_cert = _build_cert_pair("flip-client", root_cert, root_pri_key) + +with open(os.path.join(dest_dir, "root", f"rootCA.crt"), "wb") as f: + f.write(serialize_cert(root_cert)) +with open(os.path.join(dest_dir, "root", f"root.key"), "wb") as f: + f.write(serialize_pri_key(root_pri_key)) +with open(os.path.join(dest_dir, "server", f"server.crt"), "wb") as f: + f.write(serialize_cert(server_cert)) +with open(os.path.join(dest_dir, "server", f"server.key"), "wb") as f: + f.write(serialize_pri_key(server_pri_key)) +with open(os.path.join(dest_dir, "client", f"client.crt"), "wb") as f: + f.write(serialize_cert(client_cert)) +with open(os.path.join(dest_dir, "client", f"client.key"), "wb") as f: + f.write(serialize_pri_key(client_pri_key)) diff --git a/build_utils/generate_signatures.py b/build_utils/generate_signatures.py new file mode 100644 index 0000000000..e646a9785d --- /dev/null +++ b/build_utils/generate_signatures.py @@ -0,0 +1,35 @@ +import os +import pickle +import datetime + +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.backends import default_backend + + +def sign_all(content_folder, signing_pri_key): + signatures = dict() + for f in os.listdir(content_folder): + path = os.path.join(content_folder, f) + if os.path.isfile(path): + signature = signing_pri_key.sign( + data=open(path, "rb").read(), + padding=padding.PSS( + mgf=padding.MGF1(hashes.SHA256()), + salt_length=padding.PSS.MAX_LENGTH, + ), + algorithm=hashes.SHA256(), + ) + signatures[f] = signature + return signatures + + +dest_dir = os.path.join(os.getcwd(), "app", "server", "startup") +root_pri_key_path = os.path.join(os.getcwd(), "root-private-key.pem") +with open(root_pri_key_path, 'rb') as pem_in: + pemlines = pem_in.read() +root_pri_key = serialization.load_pem_private_key(pemlines, password=None, backend=default_backend()) + +signatures = sign_all(dest_dir, root_pri_key) + +pickle.dump(signatures, open(os.path.join(dest_dir, "signature.pkl"), "wb")) diff --git a/build_utils/project.yml b/build_utils/project.yml new file mode 100644 index 0000000000..71793ff8c1 --- /dev/null +++ b/build_utils/project.yml @@ -0,0 +1,39 @@ +api_version: 2 +builders: +- args: + template_file: flip_template.yml + path: build_utils.workspace.WorkspaceBuilder +- path: nvflare.lighter.impl.template.TemplateBuilder +- args: + admin_host: localhost + admin_port: 8003 + client_host: api-private.flip.aicentre.co.uk + client_port: 30021 + config_folder: config + server_host: 0.0.0.0 + server_port: 8002 + path: build_utils.static_file.StaticFileBuilder +- path: build_utils.cert.CertBuilder +- args: + coeff_mod_bit_sizes: + - 60 + - 40 + - 40 + poly_modulus_degree: 8192 + scale_bits: 40 + scheme: CKKS + path: nvflare.lighter.impl.he.HEBuilder +- path: nvflare.lighter.impl.signature.SignatureBuilder +description: flip-project +name: flip +participants: +- admin_port: 8003 + enable_byoc: true + fed_learn_port: 8002 + name: server + org: nvidia + type: server +- enable_byoc: true + name: client + org: nvidia + type: client diff --git a/build_utils/static_file.py b/build_utils/static_file.py new file mode 100644 index 0000000000..ebaaaf2a1b --- /dev/null +++ b/build_utils/static_file.py @@ -0,0 +1,207 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from nvflare.lighter.spec import Builder +from nvflare.lighter.utils import sh_replace + + +class StaticFileBuilder(Builder): + def __init__(self, enable_byoc=False, config_folder="", app_validator="", docker_image="", client_port=30021, client_host="", + server_host="", server_port=8002, admin_host="", admin_port=8003): + """Build all static files from template. + + Uses the information from project.yml through study to go through the participants and write the contents of + each file with the template, and replacing with the appropriate values from project.yml. + + Usually, two main categories of files are created in all FL participants, static and dynamic. Static files + have similar contents among different participants, with small differences. For example, the differences in + sub_start.sh are client name and python module. Those are basically static files. This builder uses template + file and string replacement to generate those static files for each participant. + + Args: + enable_byoc: for each participant, true to enable loading of code in the custom folder of applications + config_folder: usually "config" + app_validator: optional path to an app validator to verify that uploaded app has the expected structure + docker_image: when docker_image is set to a docker image name, docker.sh will be generated on server/client/admin + """ + self.enable_byoc = enable_byoc + self.config_folder = config_folder + self.docker_image = docker_image + self.app_validator = app_validator + self.client_port = client_port + self.client_host = client_host + self.server_host = server_host + self.server_port = server_port + self.admin_host = admin_host + self.admin_port = admin_port + + + def _write(self, file_full_path, content, mode, exe=False): + mode = mode + "w" + with open(file_full_path, mode) as f: + f.write(content) + if exe: + os.chmod(file_full_path, 0o755) + + def _build_server(self, server, ctx): + config = json.loads(self.template["fed_server"]) + dest_dir = self.get_kit_dir(server, ctx) + server_0 = config["servers"][0] + server_0["name"] = self.study_name + admin_port = self.admin_port + ctx["admin_port"] = admin_port + fed_learn_port = self.server_port + ctx["fed_learn_port"] = fed_learn_port + ctx["server_name"] = server.name + server_0["service"]["target"] = f"{self.server_host}:{fed_learn_port}" + server_0["admin_host"] = self.admin_host + server_0["admin_port"] = admin_port + config["enable_byoc"] = server.enable_byoc + if self.app_validator: + config["app_validator"] = {"path": self.app_validator} + self._write(os.path.join(dest_dir, "fed_server.json"), json.dumps(config), "t") + replacement_dict = { + "admin_port": admin_port, + "fed_learn_port": fed_learn_port, + "config_folder": self.config_folder, + "docker_image": self.docker_image, + } + if self.docker_image: + self._write( + os.path.join(dest_dir, "docker.sh"), + sh_replace(self.template["docker_svr_sh"], replacement_dict), + "t", + exe=True, + ) + self._write( + os.path.join(dest_dir, "start.sh"), + self.template["start_svr_sh"], + "t", + exe=True, + ) + self._write( + os.path.join(dest_dir, "sub_start.sh"), + sh_replace(self.template["sub_start_svr_sh"], replacement_dict), + "t", + exe=True, + ) + self._write( + os.path.join(dest_dir, "log.config"), + self.template["log_config"], + "t", + ) + self._write( + os.path.join(dest_dir, "readme.txt"), + self.template["readme_fs"], + "t", + ) + self._write( + os.path.join(dest_dir, "stop_fl.sh"), + self.template["stop_fl_sh"], + "t", + exe=True, + ) + + def _build_client(self, client, ctx): + config = json.loads(self.template["fed_client"]) + dest_dir = self.get_kit_dir(client, ctx) + config["servers"][0]["service"]["target"] = f"{self.client_host}:{self.client_port}" + config["servers"][0]["name"] = self.study_name + config["enable_byoc"] = client.enable_byoc + replacement_dict = { + "client_name": f"{client.subject}", + "config_folder": self.config_folder, + "docker_image": self.docker_image, + } + + self._write(os.path.join(dest_dir, "fed_client.json"), json.dumps(config), "t") + if self.docker_image: + self._write( + os.path.join(dest_dir, "docker.sh"), + sh_replace(self.template["docker_cln_sh"], replacement_dict), + "t", + exe=True, + ) + self._write( + os.path.join(dest_dir, "start.sh"), + self.template["start_cln_sh"], + "t", + exe=True, + ) + self._write( + os.path.join(dest_dir, "sub_start.sh"), + sh_replace(self.template["sub_start_cln_sh"], replacement_dict), + "t", + exe=True, + ) + self._write( + os.path.join(dest_dir, "log.config"), + self.template["log_config"], + "t", + ) + self._write( + os.path.join(dest_dir, "readme.txt"), + self.template["readme_fc"], + "t", + ) + self._write( + os.path.join(dest_dir, "stop_fl.sh"), + self.template["stop_fl_sh"], + "t", + exe=True, + ) + + def _build_admin(self, admin, ctx): + dest_dir = self.get_kit_dir(admin, ctx) + admin_port = ctx.get("admin_port") + server_name = ctx.get("server_name") + + replacement_dict = { + "cn": f"{server_name}", + "admin_port": f"{admin_port}", + "docker_image": self.docker_image, + } + if self.docker_image: + self._write( + os.path.join(dest_dir, "docker.sh"), + sh_replace(self.template["docker_adm_sh"], replacement_dict), + "t", + exe=True, + ) + self._write( + os.path.join(dest_dir, "fl_admin.sh"), + sh_replace(self.template["fl_admin_sh"], replacement_dict), + "t", + exe=True, + ) + self._write( + os.path.join(dest_dir, "readme.txt"), + self.template["readme_am"], + "t", + ) + + def build(self, study, ctx): + self.template = ctx.get("template") + server = study.get_participants_by_type("server") + self.study_name = study.name + self._build_server(server, ctx) + + for client in study.get_participants_by_type("client", first_only=False): + self._build_client(client, ctx) + + for admin in study.get_participants_by_type("admin", first_only=False): + self._build_admin(admin, ctx) diff --git a/build_utils/workspace.py b/build_utils/workspace.py new file mode 100644 index 0000000000..16b3813198 --- /dev/null +++ b/build_utils/workspace.py @@ -0,0 +1,119 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pathlib +import shutil +import subprocess + +from nvflare.lighter.spec import Builder, Study +from nvflare.lighter.utils import generate_password + + +class WorkspaceBuilder(Builder): + def __init__(self, template_file): + """Manages the folder structure for provisioned projects. + + Sets the template_file containing scripts and configs to put into startup folders, creates directories for the + participants, and moves the provisioned project to the final location at the end + ($WORKSPACE/$PROJECT_NAME/prod_XX). WorkspaceBuilder manages and sets the number in prod_XX by incrementing from + the last time provision was run for this project in this workspace, starting with 00 to a max of 99. + + Each time the provisioning tool runs, it requires a workspace folder in the local file system. The workspace + will have the following folder structure: + + .. code-block:: text + + $WORKSPACE/ <--- this is assigned by -w option of provision command (default is workspace) + $PROJECT_NAME/ <--- this is the name value in the project.yml file + prod_00/ <--- a new prod_NN folder is created if provision does not have any errors. + prod_01/ + ... + resources/ <--- this folder stores resources for other builders to load + state/ <--- this folder stores persistent information (such as certificates) so subsequent runs of the provision command can load the state back. + wip/ <--- this is only used during runtime, and will be removed when the provision command exits + + Args: + template_file: name of template file containing scripts and configs to put into startup folders + """ + self.template_file = template_file + + def _make_dir(self, dirs): + for dir in dirs: + if not os.path.exists(dir): + os.makedirs(dir) + + def initialize(self, ctx): + workspace_dir = ctx["workspace"] + prod_dirs = [_ for _ in os.listdir(workspace_dir) if _.startswith("prod_")] + last = -1 + for dir in prod_dirs: + stage = int(dir.split("_")[-1]) + if stage > last: + last = stage + ctx["last_prod_stage"] = last + template_file_full_path = os.path.join(self.get_resources_dir(ctx), self.template_file) + file_path = pathlib.Path(__file__).parent.absolute() + shutil.copyfile(os.path.join(file_path, self.template_file), template_file_full_path) + ctx["template_file"] = self.template_file + + def build(self, study: Study, ctx: dict): + dirs = [self.get_kit_dir(p, ctx) for p in study.participants] + print(dirs) + self._make_dir(dirs) + + def finalize(self, ctx: dict): + if ctx["last_prod_stage"] >= 99: + print(f"Please clean up {ctx['workspace']} by removing prod_N folders") + print("After clean-up, rerun the provision command.") + else: + current_prod_stage = str(ctx["last_prod_stage"] + 1).zfill(2) + current_prod_dir = ctx["workspace"] + wip_dir = self.get_wip_dir(ctx) + for folder in os.listdir(wip_dir): + shutil.move(os.path.join(wip_dir, folder), ctx["workspace"]) + shutil.rmtree(wip_dir) + shutil.rmtree(self.get_resources_dir(ctx)) + shutil.rmtree(self.get_state_dir(ctx)) + ctx.pop("wip_dir", None) + print(f"Generated results can be found under {current_prod_dir}. Builder's wip folder removed.") + ctx["current_prod_dir"] = current_prod_dir + + +class DistributionBuilder(Builder): + def __init__(self, zip_password=False): + """Build the zip files for each folder. + + Creates the zip files containing the archives for each startup kit. It will add password protection if the + argument (zip_password) is true. + + Args: + zip_password: if true, will create zipped packages with passwords + """ + self.zip_password = zip_password + + def build(self, study: Study, ctx: dict): + wip_dir = self.get_wip_dir(ctx) + dirs = [name for name in os.listdir(wip_dir) if os.path.isdir(os.path.join(wip_dir, name))] + for dir in dirs: + dest_zip_file = os.path.join(wip_dir, f"{dir}") + if self.zip_password: + pw = generate_password() + run_args = ["zip", "-rq", "-P", pw, dest_zip_file + ".zip", ".", "-i", "startup/*"] + os.chdir(dest_zip_file) + subprocess.run(run_args) + os.chdir(os.path.join(dest_zip_file, "..")) + print(f"Password {pw} on {dir}.zip") + else: + shutil.make_archive(dest_zip_file, "zip", root_dir=os.path.join(wip_dir, dir), base_dir="startup") diff --git a/flip-provisioner/cert.py b/flip-provisioner/cert.py new file mode 100644 index 0000000000..374e237925 --- /dev/null +++ b/flip-provisioner/cert.py @@ -0,0 +1,157 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import pickle + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID + +from nvflare.lighter.spec import Builder + + +def serialize_pri_key(pri_key): + return pri_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + + +def serialize_cert(cert): + return cert.public_bytes(serialization.Encoding.PEM) + + +class CertBuilder(Builder): + def __init__(self): + """Build certificate chain for every participant. + + Handles building (creating and self-signing) the root CA certificates, creating server, client and + admin certificates, and having them signed by the root CA for secure communication. If the state folder has + information about previously generated certs, it loads them back and reuses them. + """ + self.root_cert = None + self.persistent_state = dict() + + def initialize(self, ctx): + state_dir = self.get_state_dir(ctx) + cert_file = os.path.join(state_dir, "cert.pkl") + if os.path.exists(cert_file): + self.persistent_state = pickle.load(open(cert_file, "rb")) + self.serialized_cert = self.persistent_state["root_cert"] + self.root_cert = x509.load_pem_x509_certificate(self.serialized_cert, default_backend()) + self.pri_key = serialization.load_pem_private_key( + self.persistent_state["root_pri_key"], password=None, backend=default_backend() + ) + self.pub_key = self.pri_key.public_key() + self.subject = self.root_cert.subject + + def _build_root(self, subject): + if not self.persistent_state: + pri_key, pub_key = self._generate_keys() + self.subject = self._x509_name(subject) + issuer = self.subject + self.root_cert = self._generate_cert(self.subject, issuer, pri_key, pub_key, ca=True) + self.pri_key = pri_key + self.pub_key = pub_key + self.serialized_cert = serialize_cert(self.root_cert) + self.persistent_state["root_cert"] = self.serialized_cert + self.persistent_state["root_pri_key"] = serialize_pri_key(self.pri_key) + + def _build_write_cert_pair(self, participant, base_name, ctx): + subject = participant.subject + if self.persistent_state and subject in self.persistent_state: + cert = x509.load_pem_x509_certificate(self.persistent_state[subject]["cert"], default_backend()) + pri_key = serialization.load_pem_private_key( + self.persistent_state[subject]["pri_key"], password=None, backend=default_backend() + ) + else: + pri_key, cert = self.get_pri_key_cert(participant) + self.persistent_state[subject] = dict(cert=serialize_cert(cert), pri_key=serialize_pri_key(pri_key)) + dest_dir = self.get_kit_dir(participant, ctx) + with open(os.path.join(dest_dir, f"{base_name}.crt"), "wb") as f: + f.write(serialize_cert(cert)) + with open(os.path.join(dest_dir, f"{base_name}.key"), "wb") as f: + f.write(serialize_pri_key(pri_key)) + with open(os.path.join(dest_dir, "rootCA.pem"), "wb") as f: + f.write(self.serialized_cert) + + def build(self, study, ctx): + self._build_root(study.name) + ctx["root_cert"] = self.root_cert + ctx["root_pri_key"] = self.pri_key + server = study.get_participants_by_type("server") + self._build_write_cert_pair(server, "server", ctx) + + for client in study.get_participants_by_type("client", first_only=False): + self._build_write_cert_pair(client, "client", ctx) + + for admin in study.get_participants_by_type("admin", first_only=False): + self._build_write_cert_pair(admin, "client", ctx) + + def get_pri_key_cert(self, participant): + pri_key, pub_key = self._generate_keys() + subject = self._x509_name(participant.subject) + issuer = self.subject + cert = self._generate_cert(subject, issuer, self.pri_key, pub_key) + return pri_key, cert + + def _generate_keys(self): + pri_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) + pub_key = pri_key.public_key() + return pri_key, pub_key + + def _generate_cert(self, subject, issuer, signing_pri_key, subject_pub_key, valid_days=360, ca=False): + builder = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(subject_pub_key) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after( + # Our certificate will be valid for 360 days + datetime.datetime.utcnow() + + datetime.timedelta(days=valid_days) + # Sign our certificate with our private key + ) + ) + if ca: + builder = ( + builder.add_extension( + x509.SubjectKeyIdentifier.from_public_key(subject_pub_key), + critical=False, + ) + .add_extension( + x509.AuthorityKeyIdentifier.from_issuer_public_key(subject_pub_key), + critical=False, + ) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False) + ) + return builder.sign(signing_pri_key, hashes.SHA256(), default_backend()) + + def _x509_name(self, cn_name, org_name=None): + name = [x509.NameAttribute(NameOID.COMMON_NAME, cn_name)] + if org_name is not None: + name.append(x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)) + return x509.Name(name) + + def finalize(self, ctx): + state_dir = self.get_state_dir(ctx) + cert_file = os.path.join(state_dir, "cert.pkl") + pickle.dump(self.persistent_state, open(cert_file, "wb")) diff --git a/flip-provisioner/project.yml b/flip-provisioner/project.yml index 0fbd934293..487df28e45 100644 --- a/flip-provisioner/project.yml +++ b/flip-provisioner/project.yml @@ -51,14 +51,11 @@ builders: # allow_byoc: true # allow_custom_datalist: true # disabled: false - - path: nvflare.lighter.impl.cert.CertBuilder + - path: flip-provisioner.cert.CertBuilder - path: nvflare.lighter.impl.he.HEBuilder args: poly_modulus_degree: 8192 coeff_mod_bit_sizes: [60, 40, 40] scale_bits: 40 scheme: CKKS - - path: nvflare.lighter.impl.signature.SignatureBuilder - - path: nvflare.lighter.impl.workspace.DistributionBuilder - args: - zip_password: false + - path: nvflare.lighter.impl.signature.SignatureBuilder \ No newline at end of file diff --git a/root-private-key.pem b/root-private-key.pem new file mode 100644 index 0000000000..0b7500a5fa --- /dev/null +++ b/root-private-key.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAtk6QPpLs +rcO99+KFCk2uyTMnLEdOBZzRsNKbA5AwnsaEwSB+hKjaxUfRU8uYSZy +yQ2SwXHpCoggt5EDkzToXC1a0+H+JZxk/gEAKkQ9A98Y050HnhRhUFW +juXyKV75GvZUidQ2s9XWMZb4x399R3FQfSJD9IPQ2zeva/biOet9DjsN +K4N0nhwXxWJEXuVoYEONG3YJPqnVe024Cc0TUD6fDGK57CflsxVbrJp +YJD9fXKN6/g9JM7BD7AaUPwP587vJtgGqQKnh4DiLzXrndwDa0HGPc5 +M4MLxsDvtu3VfgWgOPlxkj65HB9IWnsg89HzwwCxA+8zdC8cmh8T0CV +lIwIDAQABAoIBAEuUzbuLiUFPNjqHeJW5eB3lLbfvRfsQtWTEI8054d +Zpwk8ZLCrOoIiKyk+dyGC4bLWBEJb+fxKTJoeLmfA1xakHRRcDNRF/x +7Jhmp/iPZKb44H+8j/dQrrvmmfpvFSqxxdMIBr74mCNvdVf3LrsJISd +DfPGIqphPEhd9SZx8RMOe1+18046nYHXTIz4FZRdMxYtH55p8Yj9UF8Y +lMg5Xqshla6smS8G/z3u+aGWwxmbBRKJkWIBOwm+6QLU/20haVC5NFy +4wYT6He3KLWFFDcnX3BOIpfAddKl6lYSlwIMtvka9G+Pp/QKV9brHU0 +uuuvs7LnC25L2ovwbCGj04Xm0CgYEA/iMJZRm+regsraCFjvIp9ZQwa +ZgfGiVqr0LleJVTvve3RauMWvWP0yD+znjpU1eXiEdHLYE/kROxjo4n +bPlTsdYYnZbzXzp1Qul+CmBD+AMvq9rz2d5/pfdFiG1f/FOF5aacl6h +2JGO/j5wTDeZ2+/f4FuEegMmvNu/pnOIqy3cCgYEAt6S3awRs7W8gkq +rRhSpNrM06OXzbwmgi09Df4q6qow/3qbM+p/ckmd8RTSKsIEHmVAO/D +KvZJQiLjFZvhdJwblNi5gk2b5JW0KZUZA7F25NmLKRhgyzwGBpw5xbyM +hT41a3PXWMp0jxTCWTJ8Jr2eepvf3o75GnhCTkw4XwHRrUCgYEApH2j +EBg4zjFv8bXrnJlrxTN0nnRC9VZwlh4TjxgAS6PksSiXQSyeOgaqRpQ +GWdZhE2knTfHRZiK8E0w5gpRissGjzySsQ9//lB6zYR3sAHR3f6HJoi +QctRy2mdZJgXXCPeG4i4UTavxROi/IOYAJVY+ruYtehfeE/GcO62eMU +YMCgYAQicby3k0o/M8EJloEEhE4tKq/rpzj3xjz1mWzNmOrl66IG9pS +qhEjwlD+j1bM+OJTQpsb+wXtLd3TX51fgdoaNWw8pyu91IH4/GGxORt +ccOPMtWkKtStSIF92kDxzgiv+ok6AbIPBqSkZOXhuMKZtyE0OP+XQ5jA +b8oiSZecJ2QKBgGPLnCwV+B6jSowPNvpSqmR+5sLqrq68rJCcpH0Ukc +8C5kQJjC478AGMBxS4fIc5jyrlGb00XdEqXXW3LWAnCo/C3w7LTVKHS +zPLL7rAqVstWBQK4XZGo4TwyDdtcWAk5OYExAZFeaBEgJAvw1OaUXwT +o3ezsW+Jq/F3uI3ggmj9 +-----END RSA PRIVATE KEY----- \ No newline at end of file