Skip to content

Commit b675e85

Browse files
ehanson8ehanson8
committed
updates
1 parent 53c562b commit b675e85

8 files changed

+429
-106
lines changed

‎README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@ The 'filePath' is directory into which output files will be written and 'handleP
1515

1616
If you are using both a development server and a production server, you can create a separate secrets.py file with a different name (e.g. secretsProd.py) and containing the production server information. When running each of these scripts, you will be prompted to enter the file name (e.g 'secretsProd' without '.py') of an alternate secrets file. If you skip the prompt or incorrectly type the file name, the scripts will default to the information in the secrets.py file. This ensures that you will only edit the production server if you really intend to.
1717

18-
**Note**: All of these scripts skip collection '4dccec82-4cfb-4583-a728-2cb823b15ef0' for local reasons. To change this, edit the following portion of the script (typically between line 27-39)
18+
**Note**: All of these scripts skip collection '45794375-6640-4efe-848e-082e60bae375' for local reasons. To change this, edit the following portion of the script (typically between line 27-39)
1919

2020

21-
Skips collection 4dccec82-4cfb-4583-a728-2cb823b15ef0:
21+
Skips collection 45794375-6640-4efe-848e-082e60bae375:
2222

2323
for j in range (0, len (collections)):
2424
collectionID = collections[j]['uuid']
25-
if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
25+
if collectionID != '45794375-6640-4efe-848e-082e60bae375':
2626
offset = 0
2727

2828

‎addNewItemsToCollection.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
parser = argparse.ArgumentParser()
1313
parser.add_argument('-d', '--directory', help='the directory of files to be ingested. optional - if not provided, the script will ask for input')
1414
parser.add_argument('-e', '--fileExtension', help='the extension of files to be ingested. optional - if not provided, the script will ask for input')
15-
parser.add_argument('-h', '--handle', help='handle of the object to retreive. optional - if not provided, the script will ask for input')
15+
parser.add_argument('-i', '--handle', help='handle of the object to retreive. optional - if not provided, the script will ask for input')
1616
args = parser.parse_args()
1717

1818
if args.uri:

‎deleteKeyFromCollection.py‎

Lines changed: 52 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,22 @@
55
import csv
66
from datetime import datetime
77
import urllib3
8+
import argparse
9+
10+
parser = argparse.ArgumentParser()
11+
parser.add_argument('-k', '--deletedKey', help='the key to be deleted. optional - if not provided, the script will ask for input')
12+
parser.add_argument('-i', '--handle', help='handle of the collection to retreive. optional - if not provided, the script will ask for input')
13+
args = parser.parse_args()
14+
15+
if args.deletedKey:
16+
deletedKey = args.deletedKey
17+
else:
18+
deletedKey = raw_input('Enter the key to be deleted: ')
19+
20+
if args.handle:
21+
handle = args.handle
22+
else:
23+
handle = raw_input('Enter collection handle: ')
824

925
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
1026

@@ -24,9 +40,6 @@
2440
filePath = secrets.filePath
2541
verify = secrets.verify
2642

27-
collectionHandle = raw_input('Enter collection handle: ')
28-
deletedKey = raw_input('Enter key to be deleted: ')
29-
3043
startTime = time.time()
3144
data = {'email':email,'password':password}
3245
header = {'content-type':'application/json','accept':'application/json'}
@@ -35,57 +48,53 @@
3548
headerFileUpload = {'accept':'application/json'}
3649
cookiesFileUpload = cookies
3750
status = requests.get(baseURL+'/rest/status', headers=header, cookies=cookies, verify=verify).json()
51+
userFullName = status['fullname']
3852
print 'authenticated'
3953

40-
itemList = []
41-
endpoint = baseURL+'/rest/handle/'+collectionHandle
54+
endpoint = baseURL+'/rest/handle/'+handle
4255
collection = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
4356
collectionID = collection['uuid']
57+
collSels = '&collSel[]=' + collectionID
58+
59+
f=csv.writer(open(filePath+'deletedValues'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'.csv', 'wb'))
60+
f.writerow(['handle']+['deletedValue']+['delete']+['post'])
4461
offset = 0
62+
recordsEdited = 0
4563
items = ''
4664
while items != []:
47-
items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
48-
while items.status_code != 200:
49-
time.sleep(5)
50-
items = requests.get(baseURL+'/rest/collections/'+str(collectionID)+'/items?limit=200&offset='+str(offset), headers=header, cookies=cookies, verify=verify)
51-
items = items.json()
52-
for k in range (0, len (items)):
53-
itemID = items[k]['uuid']
54-
itemList.append(itemID)
55-
offset = offset + 200
56-
elapsedTime = time.time() - startTime
57-
m, s = divmod(elapsedTime, 60)
58-
h, m = divmod(m, 60)
59-
print 'Item list creation time: ','%d:%02d:%02d' % (h, m, s)
60-
61-
recordsEdited = 0
62-
f=csv.writer(open(filePath+'deletedKey'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'.csv', 'wb'))
63-
f.writerow(['itemID']+['deletedKey']+['delete']+['post'])
64-
for number, itemID in enumerate(itemList):
65-
itemsRemaining = len(itemList) - number
66-
print 'Items remaining: ', itemsRemaining, 'ItemID: ', itemID
67-
metadata = requests.get(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify).json()
68-
itemMetadataProcessed = []
69-
for l in range (0, len (metadata)):
70-
if metadata[l]['key'] == deletedKey:
71-
provNote = '\''+deletedKey+'\' was deleted through a batch process on '+datetime.now().strftime('%Y-%m-%d %H:%M:%S')+'.'
72-
provNoteElement = {}
73-
provNoteElement['key'] = 'dc.description.provenance'
74-
provNoteElement['value'] = unicode(provNote)
75-
provNoteElement['language'] = 'en_US'
76-
itemMetadataProcessed.append(provNoteElement)
77-
else:
78-
itemMetadataProcessed.append(metadata[l])
79-
80-
if itemMetadataProcessed != metadata:
65+
endpoint = baseURL+'/rest/filtered-items?query_field[]='+deletedKey+'&query_op[]=exists&query_val[]='+collSels+'&limit=200&offset='+str(offset)
66+
print endpoint
67+
response = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
68+
items = response['items']
69+
for item in items:
70+
itemMetadataProcessed = []
71+
itemLink = item['link']
72+
print itemLink
73+
metadata = requests.get(baseURL + itemLink + '/metadata', headers=header, cookies=cookies, verify=verify).json()
74+
for l in range (0, len (metadata)):
75+
metadata[l].pop('schema', None)
76+
metadata[l].pop('element', None)
77+
metadata[l].pop('qualifier', None)
78+
languageValue = metadata[l]['language']
79+
if metadata[l]['key'] == deletedKey:
80+
provNote = '\''+deletedKey+'\' was deleted through a batch process on '+datetime.now().strftime('%Y-%m-%d %H:%M:%S')+'.'
81+
provNoteElement = {}
82+
provNoteElement['key'] = 'dc.description.provenance'
83+
provNoteElement['value'] = unicode(provNote)
84+
provNoteElement['language'] = 'en_US'
85+
itemMetadataProcessed.append(provNoteElement)
86+
else:
87+
itemMetadataProcessed.append(metadata[l])
8188
recordsEdited = recordsEdited + 1
8289
itemMetadataProcessed = json.dumps(itemMetadataProcessed)
83-
print 'updated', itemID, recordsEdited
84-
delete = requests.delete(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify)
90+
print 'updated', itemLink, recordsEdited
91+
delete = requests.delete(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify)
8592
print delete
86-
post = requests.put(baseURL+'/rest/items/'+str(itemID)+'/metadata', headers=header, cookies=cookies, verify=verify, data=itemMetadataProcessed)
93+
post = requests.put(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify, data=itemMetadataProcessed)
8794
print post
88-
f.writerow([itemID]+[deletedKey]+[delete]+[post])
95+
f.writerow([itemLink]+[deletedKey]+[delete]+[post])
96+
offset = offset + 200
97+
print offset
8998

9099
logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
91100

‎deleteKeyFromCommunity.py‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import json
2+
import requests
3+
import secrets
4+
import time
5+
import csv
6+
from datetime import datetime
7+
import urllib3
8+
import argparse
9+
10+
parser = argparse.ArgumentParser()
11+
parser.add_argument('-k', '--deletedKey', help='the key to be deleted. optional - if not provided, the script will ask for input')
12+
parser.add_argument('-i', '--handle', help='handle of the community to retreive. optional - if not provided, the script will ask for input')
13+
args = parser.parse_args()
14+
15+
if args.deletedKey:
16+
deletedKey = args.deletedKey
17+
else:
18+
deletedKey = raw_input('Enter the key to be deleted: ')
19+
20+
if args.handle:
21+
handle = args.handle
22+
else:
23+
handle = raw_input('Enter collection handle: ')
24+
25+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
26+
27+
secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
28+
if secretsVersion != '':
29+
try:
30+
secrets = __import__(secretsVersion)
31+
print 'Editing Production'
32+
except ImportError:
33+
print 'Editing Stage'
34+
35+
baseURL = secrets.baseURL
36+
email = secrets.email
37+
password = secrets.password
38+
filePath = secrets.filePath
39+
verify = secrets.verify
40+
41+
startTime = time.time()
42+
data = {'email':email,'password':password}
43+
header = {'content-type':'application/json','accept':'application/json'}
44+
session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, params=data).cookies['JSESSIONID']
45+
cookies = {'JSESSIONID': session}
46+
headerFileUpload = {'accept':'application/json'}
47+
cookiesFileUpload = cookies
48+
status = requests.get(baseURL+'/rest/status', headers=header, cookies=cookies, verify=verify).json()
49+
userFullName = status['fullname']
50+
print 'authenticated'
51+
52+
endpoint = baseURL+'/rest/handle/'+handle
53+
community = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
54+
communityID = community['uuid']
55+
collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
56+
collSels = ''
57+
for j in range (0, len (collections)):
58+
collectionID = collections[j]['uuid']
59+
collSel = '&collSel[]=' + collectionID
60+
collSels = collSels + collSel
61+
62+
f=csv.writer(open(filePath+'deletedValues'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'.csv', 'wb'))
63+
f.writerow(['handle']+['deletedValue']+['delete']+['post'])
64+
offset = 0
65+
recordsEdited = 0
66+
items = ''
67+
while items != []:
68+
endpoint = baseURL+'/rest/filtered-items?query_field[]='+deletedKey+'&query_op[]=exists&query_val[]='+collSels+'&limit=200&offset='+str(offset)
69+
print endpoint
70+
response = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
71+
items = response['items']
72+
for item in items:
73+
itemMetadataProcessed = []
74+
itemLink = item['link']
75+
print itemLink
76+
metadata = requests.get(baseURL + itemLink + '/metadata', headers=header, cookies=cookies, verify=verify).json()
77+
for l in range (0, len (metadata)):
78+
metadata[l].pop('schema', None)
79+
metadata[l].pop('element', None)
80+
metadata[l].pop('qualifier', None)
81+
languageValue = metadata[l]['language']
82+
if metadata[l]['key'] == deletedKey:
83+
provNote = '\''+deletedKey+'\' was deleted through a batch process on '+datetime.now().strftime('%Y-%m-%d %H:%M:%S')+'.'
84+
provNoteElement = {}
85+
provNoteElement['key'] = 'dc.description.provenance'
86+
provNoteElement['value'] = unicode(provNote)
87+
provNoteElement['language'] = 'en_US'
88+
itemMetadataProcessed.append(provNoteElement)
89+
else:
90+
itemMetadataProcessed.append(metadata[l])
91+
recordsEdited = recordsEdited + 1
92+
itemMetadataProcessed = json.dumps(itemMetadataProcessed)
93+
print 'updated', itemLink, recordsEdited
94+
delete = requests.delete(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify)
95+
print delete
96+
post = requests.put(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify, data=itemMetadataProcessed)
97+
print post
98+
f.writerow([itemLink]+[deletedKey]+[delete]+[post])
99+
offset = offset + 200
100+
print offset
101+
102+
logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
103+
104+
elapsedTime = time.time() - startTime
105+
m, s = divmod(elapsedTime, 60)
106+
h, m = divmod(m, 60)
107+
print 'Total script run time: ', '%d:%02d:%02d' % (h, m, s)

‎replaceValueInCollection.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import json
2+
import requests
3+
import secrets
4+
import csv
5+
import time
6+
import urllib3
7+
import argparse
8+
from datetime import datetime
9+
10+
parser = argparse.ArgumentParser()
11+
parser.add_argument('-k', '--key', help='the key to be searched. optional - if not provided, the script will ask for input')
12+
parser.add_argument('-1', '--replacedValue', help='the value to be replaced. optional - if not provided, the script will ask for input')
13+
parser.add_argument('-2', '--replacementValue', help='the replacement value. optional - if not provided, the script will ask for input')
14+
parser.add_argument('-i', '--handle', help='handle of the collection to retreive. optional - if not provided, the script will ask for input')
15+
args = parser.parse_args()
16+
17+
if args.key:
18+
key = args.key
19+
else:
20+
key = raw_input('Enter the key: ')
21+
if args.replacedValue:
22+
replacedValue = args.replacedValue
23+
else:
24+
replacedValue = raw_input('Enter the value to be replaced: ')
25+
if args.replacementValue:
26+
replacementValue = args.replacementValue
27+
else:
28+
replacementValue = raw_input('Enter the replacement value: ')
29+
if args.handle:
30+
handle = args.handle
31+
else:
32+
handle = raw_input('Enter collection handle: ')
33+
34+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
35+
36+
secretsVersion = raw_input('To edit production server, enter the name of the secrets file: ')
37+
if secretsVersion != '':
38+
try:
39+
secrets = __import__(secretsVersion)
40+
print 'Editing Production'
41+
except ImportError:
42+
print 'Editing Stage'
43+
44+
baseURL = secrets.baseURL
45+
email = secrets.email
46+
password = secrets.password
47+
filePath = secrets.filePath
48+
verify = secrets.verify
49+
50+
startTime = time.time()
51+
data = {'email':email,'password':password}
52+
header = {'content-type':'application/json','accept':'application/json'}
53+
session = requests.post(baseURL+'/rest/login', headers=header, verify=verify, params=data).cookies['JSESSIONID']
54+
cookies = {'JSESSIONID': session}
55+
headerFileUpload = {'accept':'application/json'}
56+
cookiesFileUpload = cookies
57+
status = requests.get(baseURL+'/rest/status', headers=header, cookies=cookies, verify=verify).json()
58+
userFullName = status['fullname']
59+
print 'authenticated'
60+
61+
endpoint = baseURL+'/rest/handle/'+handle
62+
collection = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
63+
collectionID = collection['uuid']
64+
collSels = '&collSel[]=' + collectionID
65+
66+
f=csv.writer(open(filePath+'replacedValues'+datetime.now().strftime('%Y-%m-%d %H.%M.%S')+'.csv', 'wb'))
67+
f.writerow(['handle']+['replacedValue']+['replacementValue'])
68+
offset = 0
69+
recordsEdited = 0
70+
items = ''
71+
while items != []:
72+
endpoint = baseURL+'/rest/filtered-items?query_field[]='+key+'&query_op[]=equals&query_val[]='+replacedValue+collSels+'&limit=200&offset='+str(offset)
73+
print endpoint
74+
replacedKey = key
75+
replacementKey = key
76+
response = requests.get(endpoint, headers=header, cookies=cookies, verify=verify).json()
77+
items = response['items']
78+
for item in items:
79+
itemMetadataProcessed = []
80+
itemLink = item['link']
81+
metadata = requests.get(baseURL + itemLink + '/metadata', headers=header, cookies=cookies, verify=verify).json()
82+
for l in range (0, len (metadata)):
83+
metadata[l].pop('schema', None)
84+
metadata[l].pop('element', None)
85+
metadata[l].pop('qualifier', None)
86+
languageValue = metadata[l]['language']
87+
if metadata[l]['key'] == replacedKey and metadata[l]['value'] == replacedValue:
88+
replacedElement = metadata[l]
89+
updatedMetadataElement = {}
90+
updatedMetadataElement['key'] = replacementKey
91+
updatedMetadataElement['value'] = unicode(replacementValue)
92+
updatedMetadataElement['language'] = languageValue
93+
itemMetadataProcessed.append(updatedMetadataElement)
94+
provNote = '\''+replacedKey+': '+replacedValue+'\' was replaced by \''+replacementKey+': '+replacementValue+'\' through a batch process on '+datetime.now().strftime('%Y-%m-%d %H:%M:%S')+'.'
95+
provNoteElement = {}
96+
provNoteElement['key'] = 'dc.description.provenance'
97+
provNoteElement['value'] = unicode(provNote)
98+
provNoteElement['language'] = 'en_US'
99+
itemMetadataProcessed.append(provNoteElement)
100+
recordsEdited = recordsEdited + 1
101+
else:
102+
if metadata[l] not in itemMetadataProcessed:
103+
itemMetadataProcessed.append(metadata[l])
104+
itemMetadataProcessed = json.dumps(itemMetadataProcessed)
105+
print 'updated', itemLink, recordsEdited
106+
delete = requests.delete(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify)
107+
print delete
108+
post = requests.put(baseURL+itemLink+'/metadata', headers=header, cookies=cookies, verify=verify, data=itemMetadataProcessed)
109+
print post
110+
f.writerow([itemLink]+[updatedMetadataElement['key']]+[updatedMetadataElement['value']]+[delete]+[post])
111+
offset = offset + 200
112+
print offset
113+
114+
logout = requests.post(baseURL+'/rest/logout', headers=header, cookies=cookies, verify=verify)
115+
116+
elapsedTime = time.time() - startTime
117+
m, s = divmod(elapsedTime, 60)
118+
h, m = divmod(m, 60)
119+
print 'Total script run time: ', '%d:%02d:%02d' % (h, m, s)

‎repositoryMetadataBackup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
collections = requests.get(baseURL+'/rest/communities/'+str(communityID)+'/collections', headers=header, cookies=cookies, verify=verify).json()
4444
for j in range (0, len (collections)):
4545
collectionID = collections[j]['uuid']
46-
if collectionID != '4dccec82-4cfb-4583-a728-2cb823b15ef0':
46+
if collectionID != '45794375-6640-4efe-848e-082e60bae375':
4747
collectionHandle = collections[j]['handle'].replace(handlePrefix,'').replace('/','-')
4848
print 'collectionID: ', collectionID
4949
itemList = []

0 commit comments

Comments
 (0)