Lissa dev

.gitignore

@@ -3,6 +3,10 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+# Local data archives
+datastore/src/data/archive/
+datastore_client/src/archive/
+
 # C extensions
 *.so
 
@@ -149,4 +153,4 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+#.idea/

datastore/requirements.txt

@@ -1,3 +1,4 @@
-flask==2.0.2
+flask==2.0.3
 gunicorn==20.1.0
-requests==2.27.0
+requests==2.27.0
+pandas==1.3.5

datastore/src/app.py

@@ -1,14 +1,286 @@
 from flask import Flask, jsonify
 from os import environ
+import os
+import pandas as pd
+import json
+import csv
 
+# from data_processing import *
+from data_loading import *
+
+
+
+## Get Parameters
 SECRET_KEY = environ.get("SECRET_KEY")
 print("SECRET KEY", SECRET_KEY)
+
+# ----------------------------------------------------------------------------
+# DATA PARAMETERS
+# ----------------------------------------------------------------------------
+current_folder = os.path.dirname(__file__)
+DATA_PATH = os.path.join(current_folder,'data')
+ASSETS_PATH = os.path.join(current_folder,'assets')
+
+
+# Path to Report files at TACC
+api_root = 'https://api.a2cps.org/files/v2/download/public/system/a2cps.storage.community/reports'
+
+# Load subjects locally
+# Opening JSON file
+filepath = os.path.join(DATA_PATH,'lsj.json')
+f = open(filepath)
+# returns JSON object as a dictionary
+subjects_raw = json.load(f)
+# Closing file
+f.close()
+
+# ----------------------------------------------------------------------------
+# LOAD ASSETS FILES
+# ----------------------------------------------------------------------------
+asset_files_dict = {
+    'screening_sites': 'screening_sites.csv',
+    'display_terms': 'A2CPS_display_terms.csv',
+}
+
+display_terms, display_terms_dict, display_terms_dict_multi = load_display_terms(ASSETS_PATH, asset_files_dict['display_terms'])
+
+screening_sites = pd.read_csv(os.path.join(ASSETS_PATH,asset_files_dict['screening_sites']))
+
+# ----------------------------------------------------------------------------
+# LOAD INITAL DATA FROM FILES
+# ----------------------------------------------------------------------------
+
+local_date = '2022-09-08'
+
+local_imaging_data = {
+    'date': local_date,
+    'data': get_local_imaging_data(DATA_PATH)}
+
+local_blood_data = {
+    'date': local_date,
+    'data': get_local_blood_data(DATA_PATH)}
+
+
+# local_subjects_json = get_local_subjects_raw(DATA_PATH)
+local_subjects_data = {
+    'date': local_date,
+    'data': process_subjects(subjects_raw,screening_sites, display_terms_dict, display_terms_dict_multi)
+    }
+
+local_data = {
+        'imaging': local_imaging_data,
+        'blood': local_imaging_data,
+        'subjects': local_subjects_data
+}
+
+# ----------------------------------------------------------------------------
+# APIS
+# ----------------------------------------------------------------------------
+datetime_format = "%m/%d/%Y, %H:%M:%S"
+
+apis_imaging_index = {}
+data_state = 'empty'
+api_data_index = {
+    'blood':'',
+    'imaging':'',
+    'subjects':'',
+    'raw': 'local'
+}
+api_request_state = {
+    'blood':None,
+    'imaging':None,
+    'subjects1':None,
+    'subjects2':None,
+}
+api_data_cache = {
+    'blood':None,
+    'imaging':None,
+    'subjects':None,
+    'raw': None
+}
+
+api_subjects = {'date':None, 'data':None}
+
 app = Flask(__name__)
 
-@app.route("/api")
-def api():
-    d = { "key": "value" }
-    return jsonify(d)
+# APIS: try to load new data, if doesn't work, get most recent
+@app.route("/api/apis")
+def api_apis():
+    print(api_data_index)
+    return jsonify(api_data_index)
+
+@app.route("/api/imaging")
+def api_imaging():
+    global datetime_format
+    global api_data_index
+    global api_data_cache
+    try:
+        if not api_data_index['imaging'] or not check_data_current(datetime.strptime(api_data_index['imaging'], datetime_format)):
+            api_date = datetime.now().strftime(datetime_format)
+            imaging_data = get_api_imaging_data()
+            if imaging_data:
+                api_data_cache['imaging'] = imaging_data
+                api_data_index['imaging'] = api_date
+        return jsonify({'date': api_data_index['imaging'], 'data': api_data_cache['imaging']})
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify('error: {}'.format(e))
+
+@app.route("/api/blood")
+def api_blood():
+    global datetime_format
+    global api_data_index
+    global api_data_cache
+    try:
+        if not api_data_index['blood'] or not check_data_current(datetime.strptime(api_data_index['blood'], datetime_format)):
+            api_date = datetime.now().strftime(datetime_format)
+            blood_data, blood_data_request_status = get_api_blood_data()
+            if blood_data:
+                api_data_index['blood'] = api_date
+                api_data_cache['blood'] = blood_data
+
+            with open('requests.csv', 'a', newline='') as f:
+                writer = csv.writer(f)
+                for i in blood_data_request_status:
+                    writer.writerow(i)
+                f.close()
+
+        return jsonify({'date': api_data_index['blood'], 'data': api_data_cache['blood']})
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify('error: {}'.format(e))
+
+
+@app.route("/api/subjects")
+def api_subjects():
+    global datetime_format
+    global api_data_index
+    global api_data_cache
+
+    try:
+        if not api_data_index['subjects'] or not check_data_current(datetime.strptime(api_data_index['subjects'], datetime_format)):
+            api_date = datetime.now().strftime(datetime_format)
+            latest_subjects_json = get_api_subjects_json()
+            if latest_subjects_json:
+                # latest_data = create_clean_subjects(latest_subjects_json, screening_sites, display_terms_dict, display_terms_dict_multi)
+                latest_data = process_subjects(latest_subjects_json,screening_sites, display_terms_dict, display_terms_dict_multi)
+
+                api_data_cache['subjects'] = latest_data
+                api_data_index['subjects'] = api_date
+
+        return jsonify({'date': api_data_index['subjects'], 'data': api_data_cache['subjects']})
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify('error: {}'.format(e))
+
+@app.route("/api/load_data")
+def api_load_data():
+    global datetime_format
+    global api_data_index
+    global api_data_cache
+
+    try:
+        # return api_data_index
+        # if not 'tester' in api_data_index.keys():
+        if not api_request_state['subjects1']  == 200 or not api_request_state['subjects2'] == 200 :
+            api_date = datetime.now().strftime(datetime_format)
+            api_data_index['raw'] = api_date
+
+            # latest_subjects_json = get_api_subjects_json()
+
+            api_root = 'https://api.a2cps.org/files/v2/download/public/system/a2cps.storage.community/reports'
+
+            # Load Json Data
+            subjects1_filepath = '/'.join([api_root,'subjects','subjects-1-latest.json'])
+            subjects1_request = requests.get(subjects1_filepath)
+            api_request_state['subjects1'] = subjects1_request.status_code
+            if subjects1_request.status_code == 200:
+                subjects1 = subjects1_request.json()
+            else:
+                subjects1 = subjects1_request.status_code
+                # return {'status':'500', 'source': api_dict['subjects']['subjects1']}
+
+            subjects2_filepath = '/'.join([api_root,'subjects','subjects-2-latest.json'])
+            subjects2_request = requests.get(subjects2_filepath)
+            api_request_state['subjects2'] = subjects2_request.status_code
+            if subjects2_request.status_code == 200:
+                subjects2 = subjects2_request.json()
+            else:
+                subjects2 = subjects2_request.status_code
+                # return {'status':'500', 'source': api_dict['subjects']['subjects2']}
+
+            # Create combined json
+            latest_subjects_json = {'1': subjects1, '2': subjects2}
+            api_data_cache['raw'] = latest_subjects_json
+
+            if latest_subjects_json:
+                api_data_cache['raw'] = latest_subjects_json
+                return jsonify(api_data_cache['raw'])
+            else:
+                api_data_cache['raw'] = 'failed'
+                return jsonify({'failed':''})
+
+
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify('error: {}'.format(e))
+
+
+@app.route("/api/tester")
+def api_tester():
+
+    global local_subjects_data
+
+    try:
+        return jsonify(local_subjects_data)
+
+
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify('error: {}'.format(e))
+
+# @app.route("/api/screening_sites")
+# def api_screening_sites():
+#     screening_site_dict = screening_sites.to_dict('records')
+#     return jsonify(screening_site_dict)
+
+# @app.route("/api/subjects")
+# def api_subjects():
+#
+#     global api_subjects_json_cache
+#     try:
+#         if not check_available_data(api_subjects_json_cache):
+#             current_date = str(datetime.now())
+#             latest_subjects_json = get_api_subjects_json()
+#             if latest_subjects_json:
+#                 latest_data = create_clean_subjects(latest_subjects_json, screening_sites, display_terms_dict, display_terms_dict_multi)
+#                 api_subjects_data = {
+#                     'date': current_date,
+#                     'data': latest_data
+#                     }
+#
+#                 api_subjects_json_cache = [api_subjects_json]
+#
+#             else:
+#                 api_subjects_json_cache = api_subjects_json_cache
+#             return jsonify(api_subjects_json_cache[-1])
+#
+#     except Exception as e:
+#         traceback.print_exc()
+#         # try:
+#         #     return jsonify(local_data['subjects'])
+#         # except:
+#         return jsonify('error: {}'.format(e))
+
+
+@app.route("/api/full")
+def api_full():
+    datafeeds = {'date': {'weekly': 'today', 'consort': 'today', 'blood': 'today'},
+                'data': {'weekly': 'tbd', 'consort': 'tbd', 'blood': 'tbd'}}
+    return jsonify(datafeeds)
+
+
+
 
 if __name__ == "__main__":
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0')

datastore/src/assets/A2CPS_display_terms.csv

@@ -0,0 +1,75 @@
+api_field,api_value,multi,data_dictionary_values,display_text,changed_text
+redcap_data_access_group,rush_university_me,0,MCC1: Rush,MCC1: Rush,0
+redcap_data_access_group,northshore,0,MCC1: NorthShore,MCC1: NorthShore,0
+redcap_data_access_group,uchicago,0,MCC1: UChicago,MCC1: UChicago,0
+redcap_data_access_group,university_of_mich,0,MCC2: UMichigan,MCC2: UMichigan,0
+redcap_data_access_group,wayne_state,0,MCC2: Wayne State,MCC2: Wayne State,0
+redcap_data_access_group,spectrum_health,0,MCC2: Spectrum Health,MCC2: Spectrum Health,0
+sp_data_site,1,0,MCC2: UMichigan,MCC2: UMichigan,0
+sp_data_site,2,0,MCC2: Wayne State,MCC2: Wayne State,0
+sp_data_site,3,0,MCC2: Spectrum Health,MCC2: Spectrum Health,0
+participation_interest,0,0,No,No,0
+participation_interest,1,0,Maybe,Maybe,0
+participation_interest,2,0,Yes,Yes,0
+screening_race,6,0,American Indian or Alaska Native,American Indian or Alaska Native,0
+screening_race,5,0,Asian,Asian,0
+screening_race,4,0,Black/African American,Black or African-American,1
+screening_race,3,0,Native Hawaiian or other Pacific Islander,Native Hawaiian or Pacific Islander,1
+screening_race,2,0,White/Caucasian,White,1
+screening_race,1,0,Other,Unknown,1
+screening_race,0,0,Prefer not to answer,Not Reported,1
+dem_race,1,0,American Indian or Alaska Native,American Indian or Alaska Native,0
+dem_race,2,0,Asian,Asian,0
+dem_race,3,0,Black or African-American,Black or African-American,0
+dem_race,4,0,Native Hawaiian or Pacific Islander,Native Hawaiian or Pacific Islander,0
+dem_race,5,0,White,White,0
+dem_race,6,0,Unknown,Unknown,0
+dem_race,7,0,Not Reported,Not Reported,0
+dem_race,8,0,Multi-Racial,Multi-Racial,0
+screening_ethnicity,2,0,Hispanic or Latino,Hispanic or Latino,0
+screening_ethnicity,1,0,Not Hispanic or Latino,Not Hispanic or Latino,0
+screening_ethnicity,0,0,Prefer not to answer,Not reported,1
+ethnic,1,0,Hispanic or Latino,Hispanic or Latino,0
+ethnic,2,0,Not Hispanic or Latino,Not Hispanic or Latino,0
+ethnic,3,0,Unknown,Unknown,0
+ethnic,4,0,Not reported,Not reported,0
+screening_gender,1,0,Male,Male,0
+screening_gender,2,0,Female,Female,0
+screening_gender,3,0,Unknown,Unknown,0
+screening_gender,4,0,Other,Other,0
+sex,1,0,Male,Male,0
+sex,2,0,Female,Female,0
+sex,3,0,Unknown,Unknown,0
+sex,4,0,Intersex ,Intersex ,0
+genident,1,0,Male,Male,0
+genident,2,0,Female,Female,0
+genident,3,0,Unknown,Unknown,0
+genident,4,0,Other,Other,0
+reason_not_interested,5,1,No specific reason,No specific reason,0
+reason_not_interested,4,1,Time-related issue or concern,Time-related issue or concern,0
+reason_not_interested,3,1,Specific study procedure,Specific study procedure,0
+reason_not_interested,2,1,Compensation insufficient,Compensation insufficient,0
+reason_not_interested,1,1,COVID-related,COVID-related,0
+reason_not_interested,0,1,Not interested in research,Not interested in research,0
+reason_not_interested,-1,1,Not provided,Not provided,0
+erep_protdev_type,1,1,Informed Consent,Informed Consent,0
+erep_protdev_type,2,1,Protocol Deviation-blood drawo,Blood Draw,1
+erep_protdev_type,3,1,Protocol Deviation-functional testing,Functional Testing,1
+erep_protdev_type,4,1,Protocol Deviation-QST,QST,1
+erep_protdev_type,5,1,Protocol Deviation-imaging,Imaging,1
+erep_protdev_type,6,1,Visit timeline (outside protocol range),Visit Timeline,1
+erep_protdev_type,7,1,Other,Other,0
+erep_ae_yn,1,1,Yes,Yes,0
+erep_ae_yn,0,1,No,No,0
+erep_ae_severity,1,1,Mild,Mild,0
+erep_ae_severity,2,1,Moderate,Moderate,0
+erep_ae_severity,3,1,Severe,Severe,0
+erep_ae_relation,1,1,Definitely Related,Definitely Related,0
+erep_ae_relation,2,1,Possibly/Probably Related,Possibly/Probably Related,0
+erep_ae_relation,3,1,Not Related,Not Related,0
+erep_ae_serious,1,1,Yes,Yes,0
+erep_ae_serious,0,1,No,No,0
+ewprimaryreason,1,0,Subject chose to discontinue the study,Subject chose to discontinue the study,0
+ewprimaryreason,2,0,Site PI chose to discontinue subject participation,Site PI chose to discontinue subject participation,0
+ewprimaryreason,3,0,"Subject is lost to follow-up, unable to locate","Subject is lost to follow-up, unable to locate",0
+ewprimaryreason,4,0,Death,Death,0