updates

Author: ehanson8

.gitignore

@@ -46,7 +46,8 @@ Network Trash Folder
 Temporary Items
 .apdisk
 secrets.py
+secretsProd.py
 *.pyc
 data/*
 !data/.keep
-.profile
+.profile

compareTwoKeysInCommunity.py

@@ -4,10 +4,21 @@
 import csv
 import time
 
+secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
+if secretsVersion != '':
+    try:
+        secrets = __import__(secretsVersion)
+        print 'Editing Production'
+    except ImportError:
+        print 'Editing Stage'
+
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
+verify = secrets.verify
+
+requests.packages.urllib3.disable_warnings()
 
 communityID = raw_input('Enter community ID: ')
 key = raw_input('Enter first key: ')
@@ -16,23 +27,23 @@
 startTime = time.time()
 data = json.dumps({'email':email,'password':password})
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, data=data).content
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
 headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
 print 'authenticated'
 
 itemList = []
 endpoint = baseURL+'/rest/communities'
-collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth).json()
+collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
 for j in range (0, len (collections)):
     collectionID = collections[j]['id']
     if collectionID != 24:
         offset = 0
         items = ''
         while items != []:
-            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth)
+            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
             while items.status_code != 200:
                 time.sleep(5)
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth)
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
             items = items.json()
             for k in range (0, len (items)):
                 itemID = items[k]['id']
@@ -47,7 +58,7 @@
 for number, itemID in enumerate(itemList):
     itemsRemaining = len(itemList) - number
     print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
     itemTuple = (itemID,)
     tupleValue1 = ''
     tupleValue2 = ''
@@ -73,7 +84,7 @@
 for i in range (0, len (valueList)):
     f.writerow([valueList[i][0]]+[valueList[i][1]]+[valueList[i][2]])
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth)
+logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)

findBogusUris.py

@@ -4,35 +4,46 @@
 import csv
 import time
 
+secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
+if secretsVersion != '':
+    try:
+        secrets = __import__(secretsVersion)
+        print 'Editing Production'
+    except ImportError:
+        print 'Editing Stage'
+
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
 handlePrefix = secrets.handlePrefix
+verify = secrets.verify
+
+requests.packages.urllib3.disable_warnings()
 
 startTime = time.time()
 data = json.dumps({'email':email,'password':password})
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, data=data).content
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
 headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
 print 'authenticated'
 
 itemList = []
 endpoint = baseURL+'/rest/communities'
-communities = requests.get(endpoint, headers=headerAuth).json()
+communities = requests.get(endpoint, headers=headerAuth, verify=verify).json()
 for i in range (0, len (communities)):
     communityID = communities[i]['id']
-    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth).json()
+    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
     for j in range (0, len (collections)):
         collectionID = collections[j]['id']
         if collectionID != 24:
             offset = 0
             items = ''
             while items != []:
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth)
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
                 while items.status_code != 200:
                     time.sleep(5)
-                    items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth)
+                    items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
                 items = items.json()
                 for k in range (0, len (items)):
                     itemID = items[k]['id']
@@ -49,14 +60,14 @@
 for number, itemID in enumerate(itemList):
     itemsRemaining = len(itemList) - number
     print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
     for l in range (0, len (metadata)):
         if metadata[l]['key'] == 'dc.identifier.uri':
             uri = str(metadata[l]['value'])
             if uri.startswith(handlePrefix) == False:
                 f.writerow([itemID]+[uri])
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth)
+logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)

findDuplicateKeys.py

@@ -4,35 +4,46 @@
 import time
 import csv
 
+secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
+if secretsVersion != '':
+    try:
+        secrets = __import__(secretsVersion)
+        print 'Editing Production'
+    except ImportError:
+        print 'Editing Stage'
+        
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
+verify = secrets.verify
+
+requests.packages.urllib3.disable_warnings()
 
 key = raw_input('Enter key: ')
 searchString = "\""+key+"\""
 
 startTime = time.time()
 data = json.dumps({'email':email,'password':password})
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, data=data).content
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
 headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
 print 'authenticated'
 
 
 itemList = []
 endpoint = baseURL+'/rest/communities'
-communities = requests.get(endpoint, headers=headerAuth).json()
+communities = requests.get(endpoint, headers=headerAuth, verify=verify).json()
 for i in range (0, len (communities)):
     communityID = communities[i]['id']
-    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth).json()
+    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
     for j in range (0, len (collections)):
         collectionID = collections[j]['id']
         if collectionID != 24:
-            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=100000', headers=headerAuth)
+            items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=100000', headers=headerAuth, verify=verify)
             while items.status_code != 200:
                 time.sleep(5)
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=100000', headers=headerAuth)
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=100000', headers=headerAuth, verify=verify)
             items = items.json()
             for k in range (0, len (items)):
                 itemID = items[k]['id']
@@ -47,12 +58,12 @@
 for number, itemID in enumerate(itemList):
     itemsRemaining = len(itemList) - number
     print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
     metadata = json.dumps(metadata)
     if metadata.find(searchString) != metadata.rfind(searchString):
         f.writerow([itemID])
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth)
+logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)

getCollectionMetadataJson.py

@@ -3,26 +3,37 @@
 import secrets
 import time
 
+secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
+if secretsVersion != '':
+    try:
+        secrets = __import__(secretsVersion)
+        print 'Editing Production'
+    except ImportError:
+        print 'Editing Stage'
+
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
+verify = secrets.verify
+
+requests.packages.urllib3.disable_warnings()
 
 handle = raw_input('Enter handle: ')
 
 data = json.dumps({'email':email,'password':password})
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, data=data).content
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
 headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
 print 'authenticated'
 startTime = time.time()
 
 endpoint = baseURL+'/rest/handle/'+handle
-collection = requests.get(endpoint, headers=headerAuth).json()
+collection = requests.get(endpoint, headers=headerAuth, verify=verify).json()
 collectionID = collection['id']
-collectionTitle = requests.get(endpoint, headers=headerAuth).json()
+collectionTitle = requests.get(endpoint, headers=headerAuth, verify=verify).json()
 endpoint = baseURL+'/rest/collections/'+str(collectionID)+'/items'
-output = requests.get(endpoint, headers=headerAuth).json()
+output = requests.get(endpoint, headers=headerAuth, verify=verify).json()
 
 itemList = []
 for i in range (0, len (output)):
@@ -33,11 +44,11 @@
 f=open(filePath+handle.replace('/','-')+'.json', 'w')
 metadataGroup = []
 for itemID in itemList:
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
     metadataGroup.append(metadata)
 json.dump(metadataGroup, f)
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth)
+logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)

getCompleteAndUniqueValuesForAllKeys.py

@@ -5,38 +5,50 @@
 import time
 import os.path
 from collections import Counter
+from datetime import datetime
+
+secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
+if secretsVersion != '':
+    try:
+        secrets = __import__(secretsVersion)
+        print 'Editing Production'
+    except ImportError:
+        print 'Editing Stage'
 
 baseURL = secrets.baseURL
 email = secrets.email
 password = secrets.password
 filePath = secrets.filePath
+verify = secrets.verify
+
+requests.packages.urllib3.disable_warnings()
 
-filePathComplete = filePath+'completeValueLists/'
-filePathUnique = filePath+'uniqueValueLists/'
+filePathComplete = filePath+'completeValueLists'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'/'
+filePathUnique = filePath+'uniqueValueLists'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'/'
 
 startTime = time.time()
 data = json.dumps({'email':email,'password':password})
 header = {'content-type':'application/json','accept':'application/json'}
-session = requests.post(baseURL+'/rest/login', headers=header, data=data).content
+session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, data=data).content
 headerAuth = {'content-type':'application/json','accept':'application/json', 'rest-dspace-token':session}
 print 'authenticated'
 
 itemList = []
 endpoint = baseURL+'/rest/communities'
-communities = requests.get(endpoint, headers=headerAuth).json()
+communities = requests.get(endpoint, headers=headerAuth, verify=verify).json()
 for i in range (0, len (communities)):
     communityID = communities[i]['id']
-    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth).json()
+    collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=headerAuth, verify=verify).json()
     for j in range (0, len (collections)):
         collectionID = collections[j]['id']
         if collectionID != 24:
             offset = 0
             items = ''
             while items != []:
-                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth)
+                items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
                 while items.status_code != 200:
                     time.sleep(5)
-                    items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth)
+                    items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=1000&offset='+str(offset), headers=headerAuth, verify=verify)
                 items = items.json()
                 for k in range (0, len (items)):
                     itemID = items[k]['id']
@@ -47,10 +59,12 @@
 h, m = divmod(m, 60)
 print 'Item list creation time: ','%d:%02d:%02d' % (h, m, s)
 
+os.mkdir(filePathComplete)
+os.mkdir(filePathUnique)
 for number, itemID in enumerate(itemList):
     itemsRemaining = len(itemList) - number
     print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
-    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth).json()
+    metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=headerAuth, verify=verify).json()
     for l in range (0, len (metadata)):
         if metadata[l]['key'] != 'dc.description.provenance':
             key = metadata[l]['key']
@@ -79,7 +93,7 @@
     for key, value in valueListCount.items():
         f.writerow([key]+[str(value).zfill(6)])
 
-logout = requests.post(baseURL+'/rest/logout', headers=headerAuth)
+logout = requests.post(baseURL+'/rest/logout', headers=headerAuth, verify=verify)
 
 elapsedTime = time.time() - startTime
 m, s = divmod(elapsedTime, 60)