updates

ehanson8 · ehanson8 · commit f32b783cdc66 · 2017-07-25T10:51:13.000-04:00
diff --git a/README.md b/README.md
@@ -7,33 +7,38 @@ All of these scripts require a secrets.py file in the same directory that must c
         password='my_dspace_password'    
         filePath = '/Users/dspace_user/dspace-data-collection/data/'
         handlePrefix = 'http://dspace.myuni.edu/handle/'
+        verify = True or False (no quotes). Use False if using an SSH tunnel to connect to the DSpace API
 ```
-The 'filePath' is directory into which output files will be written and 'handlePrefix' may or may not vary from your DSpace URL depending on your configuration. This secrets.py file will be ignored according to the repository's .gitignore file so that DSpace login details will not be inadvertently exposed through Github
+The 'filePath' is directory into which output files will be written and 'handlePrefix' may or may not vary from your DSpace URL depending on your configuration. This secrets.py file will be ignored according to the repository's .gitignore file so that DSpace login details will not be inadvertently exposed through Github.
+
+If you are using both a development server and a production server, you can create a separate secrets.py file with a different name (e.g. secretsProd.py) and containing the production server information. When running each of these scripts, you will be prompted to enter the file name (e.g 'secretsProd' without '.py') of an alternate secrets file. If you skip the prompt or incorrectly type the file name, the scripts will default to the information in the secrets.py file. This ensures that you will only access the production server if you really intend to.
+
+The command 'requests.packages.urllib3.disable_warnings()' is used to disable the excessive warnings that will be produced if the 'verify' variable is set to False, which necessary if you are using an SSH tunnel to connect to the DSpace API.
 
 **Note**: All of these scripts skip collection '24' for local reasons. To change this, edit the following portion of the script (typically between line 27-39)
 
 
-Skips collection 24: 
+Skips collection 24:
 
                 for j in range (0, len (collections)):
                         collectionID = collections[j]['id']
                         if collectionID != 24:
                         offset = 0
-            
-            
+
+
 No collections skipped:
 
                 for j in range (0, len (collections)):
                         collectionID = collections[j]['id']
                         if collectionID != 0:
                         offset = 0
-            
-            
+
+
 #### [compareTwoKeysInCommunity.py](compareTwoKeysInCommunity.py)
 Based on user input, this script extracts the values of two specified keys from a specified community to a CSV file for comparison.
 
 #### [findBogusUris.py](findBogusUris.py)
-This script extracts the item ID and the value of the key 'dc.identifier.uri' to a CSV file when the value does not begin with the handlePrefix specified in the secrets.py file. 
+This script extracts the item ID and the value of the key 'dc.identifier.uri' to a CSV file when the value does not begin with the handlePrefix specified in the secrets.py file.
 
 #### [findDuplicateKeys.py](findDuplicateKeys.py)
 Based on user input, this script extracts item IDs to a CSV file where there are multiple instances of the specified key in the item metadata.
diff --git a/allKeys.csv b/allKeys.csv
diff --git a/compareTwoKeysInCommunity.py b/compareTwoKeysInCommunity.py
@@ -11,7 +11,9 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
-
+else:
+    print 'Editing Stage'
+    
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
diff --git a/findBogusUris.py b/findBogusUris.py
@@ -11,7 +11,9 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
-
+else:
+    print 'Editing Stage'
+    
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
diff --git a/findDuplicateKeys.py b/findDuplicateKeys.py
@@ -11,7 +11,9 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
-        
+else:
+    print 'Editing Stage'
+
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
diff --git a/getCollectionMetadataJson.py b/getCollectionMetadataJson.py
@@ -10,7 +10,9 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
-
+else:
+    print 'Editing Stage'
+    
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
diff --git a/getCompleteAndUniqueValuesForAllKeys.py b/getCompleteAndUniqueValuesForAllKeys.py
@@ -14,6 +14,8 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
+else:
+    print 'Editing Stage'
 
 baseURL = secrets.baseURL
 email = secrets.email
diff --git a/getGlobalLanguageValues.py b/getGlobalLanguageValues.py
@@ -11,6 +11,8 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
+else:
+    print 'Editing Stage'
 
 baseURL = secrets.baseURL
 email = secrets.email
diff --git a/getLanguageValuesForKeys.py b/getLanguageValuesForKeys.py
@@ -4,10 +4,23 @@
 import csv
 import time
 
+secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
+if secretsVersion != '':
+    try:
+        secrets = __import__(secretsVersion)
+        print 'Editing Production'
+    except ImportError:
+        print 'Editing Stage'
+else:
+    print 'Editing Stage'
+
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
+verify = secrets.verify
+
+requests.packages.urllib3.disable_warnings()
 
 startTime = time.time()
 data = json.dumps({'email':email,'password':password})
diff --git a/getRecordsAndValuesForKey.py b/getRecordsAndValuesForKey.py
@@ -11,6 +11,8 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
+else:
+    print 'Editing Stage'
 
 baseURL = secrets.baseURL
 email = secrets.email
diff --git a/getRecordsWithKeyAndValue.py b/getRecordsWithKeyAndValue.py
@@ -11,7 +11,9 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
-
+else:
+    print 'Editing Stage'
+    
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
diff --git a/metadataCollectionsKeysMatrix.py b/metadataCollectionsKeysMatrix.py
@@ -0,0 +1,129 @@
+import json
+import requests
+import secrets
+import time
+import csv
+from collections import Counter
+
+secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
+if secretsVersion != '':
+    try:
+        secrets = __import__(secretsVersion)
+        print 'Editing Production'
+    except ImportError:
+        print 'Editing Stage'
+else:
+    print 'Editing Stage'
+
+#login info kept in secrets.py file
+baseURL = secrets.baseURL
+email = secrets.email
+password = secrets.password
+filePath = secrets.filePath
+verify = secrets.verify
+
+requests.packages.urllib3.disable_warnings()
+
+#authentication
+data = json.dumps({'email':email,'password':password})
+header = {'content-type':'application/json','accept':'application/json'}
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
+headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
+print 'authenticated'
+startTime = time.time()
+
+endpoint = baseURL+'/rest/communities'
+communities = requests.get(endpoint, headers=headerAuth, verify=verify).json()
+
+#create list of all item IDs
+itemList = []
+for i in range (0, len (communities)):
+    communityID = communities[i]['id']
+    communityName = communities[i]['name'].encode('utf-8')
+    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
+    for j in range (0, len (collections)):
+        collectionID = collections[j]['id']
+        collectionName = collections[j]['name'].encode('utf-8')
+        fullName = communityName+' - '+collectionName
+        if collectionID == 24:
+            print 'Levy Collection - skipped'
+        else:
+            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=5000', headers=headerAuth, verify=verify)
+            while items.status_code != 200:
+                time.sleep(5)
+                print 'collection:', collectionID, '# of items:',len(items), 'fail'
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=5000', headers=headerAuth, verify=verify)
+            items = items.json()
+            print 'collection:', collectionID,', Number of items:',len(items)
+            for i in range (0, len (items)):
+                itemID = items[i]['id']
+                itemList.append(itemID)
+
+#retrieve metadata from all items
+keyList = []
+for itemID in itemList:
+    print itemID
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
+    for i in range (0, len (metadata)):
+        key = metadata[i]['key']
+        if key not in keyList:
+            keyList.append(key)
+
+keyListHeader = ['collectionNameColumn']
+keyList.sort()
+keyListHeader = keyListHeader + keyList
+f=csv.writer(open('collectionsKeysMatrix.csv', 'wb'))
+f.writerow(keyListHeader)
+
+for i in range (0, len (communities)):
+    communityID = communities[i]['id']
+    communityName = communities[i]['name'].encode('utf-8')
+    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
+    for j in range (0, len (collections)):
+        collectionID = collections[j]['id']
+        if collectionID == 24:
+            print 'Levy Collection - skipped'
+        else:
+            collectionItemList = []
+            collectionName = collections[j]['name'].encode('utf-8')
+            fullName = communityName+' - '+collectionName
+            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=5000', headers=headerAuth, verify=verify)
+            while items.status_code != 200:
+                time.sleep(5)
+                print 'collection:', collectionID, '# of items:',len(items), 'fail'
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=5000', headers=headerAuth, verify=verify)
+            items = items.json()
+            for i in range (0, len (items)):
+                itemID = items[i]['id']
+                collectionItemList.append(itemID)
+
+            collectionKeyCount = {}
+            for key in keyList:
+                collectionKeyCount[key] = 0
+            for itemID in collectionItemList:
+                print itemID
+                metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
+                for i in range (0, len (metadata)):
+                    itemKey = metadata[i]['key']
+                    for key in keyList:
+                        if itemKey == key:
+                            collectionKeyCount[key] = collectionKeyCount[key]+1
+
+            collectionKeyCountList = []
+            for k, v in collectionKeyCount.items():
+                collectionKeyCountList.append(k+' '+str(v))
+            collectionKeyCountList.sort()
+            updatedCollectionKeyCountList = []
+            for entry in collectionKeyCountList:
+                count = entry[entry.index(' ')+1:]
+                updatedCollectionKeyCountList.append(count)
+            fullName = [fullName]
+            updatedCollectionKeyCountList = fullName + updatedCollectionKeyCountList
+            f.writerow(updatedCollectionKeyCountList)
+
+elapsedTime = time.time() - startTime
+m, s = divmod(elapsedTime, 60)
+h, m = divmod(m, 60)
+print "%d:%02d:%02d" % (h, m, s)
+
+logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
diff --git a/metadataOverview.py b/metadataOverview.py
@@ -12,6 +12,8 @@
         print 'Editing Production'
     except ImportError:
         print 'Editing Stage'
+else:
+    print 'Editing Stage'
 
 #login info kept in secrets.py file
 baseURL = secrets.baseURL
