updates

Author: ehanson8

.gitignore

@@ -49,4 +49,5 @@ secrets.py
 *.pyc
 data/*
 !data/.keep
-.profile
+.profile
+*.csv

README.md

@@ -8,6 +8,9 @@ More information about WSKeys is available [here](https://www.oclc.org/developer
 #### [oclcIsbn.py](oclcIsbn.py)
 This script retrieves OCLC numbers and titles based on a text file of ISBNs.
 
+#### [oclcTitleBorrowDirect.py](oclcTitleBorrowDirect.py)
+This script retrieves OCLC data based on a CSV from the BorrowDirect Data Repository (Beta), on the Penn Library Data Farm.
+
 #### [oclcTitlePhraseEnhanced.py](oclcTitlePhraseEnhanced.py)
 This script retrieves OCLC records based on a text file of titles and extracts the title, URL, author, publisher, encoding level, language, and date.
 

oclcSearchForNewNum.py

@@ -0,0 +1,41 @@
+import requests
+from bs4 import BeautifulSoup
+import csv
+import secrets
+import time
+
+startTime = time.time()
+
+baseURL = 'http://www.worldcat.org/webservices/catalog/content/'
+wskey = secrets.wskey
+f=csv.writer(open('newOclcNumResults.csv', 'wb'))
+f.writerow(['bibNum']+['search']+['newOclcNum'])
+
+filename = 'noHathiTrustMatch.csv'
+
+with open(filename) as csvfile:
+    reader = csv.DictReader(csvfile)
+    counter = 0
+    for row in reader:
+        counter = counter + 1
+        print counter
+        search = row['oclcNum']
+        bibNum = row['bibNum']
+        try:
+            response = requests.get(baseURL+search.strip()+'?wskey='+wskey).content
+            record = BeautifulSoup(response, "lxml").find('record')
+            oclcNum = record.find('controlfield', {'tag' : '001'}).text.lstrip('0')
+        except:
+            fullTitle = ''
+            oclcNum = ''
+        if search.lstrip('0') != oclcNum:
+            print search, oclcNum
+            f.writerow([bibNum]+[search]+[oclcNum])
+        else:
+            oclcNum = ''
+            f.writerow([bibNum]+[search]+[oclcNum])
+
+elapsedTime = time.time() - startTime
+m, s = divmod(elapsedTime, 60)
+h, m = divmod(m, 60)
+print 'Total script run time: ', '%d:%02d:%02d' % (h, m, s)

oclcTitlePhraseBorrowDirect.py

@@ -0,0 +1,102 @@
+import requests
+from bs4 import BeautifulSoup
+import csv
+import secrets
+import urllib
+import re
+import time
+
+startTime = time.time()
+
+fileName = raw_input('Enter file name: ')
+fileNameWithoutExtension = fileName[:fileName.index('.')]
+
+baseURL = 'http://www.worldcat.org/webservices/catalog/search/opensearch?q='
+baseURL2 = 'http://www.worldcat.org/webservices/catalog/content/'
+
+wskey = secrets.wskey
+f=csv.writer(open(fileNameWithoutExtension+'oclcSearchMatches.csv', 'wb'))
+f.writerow(['searchOclcNum']+['borrower']+['lender']+['status']+['patronType']+['isbn']+['searchTitle']+['searchAuthor']+['searchDate']+['oclcNum']+['oclcTitle']+['oclcAuthor']+['oclcPublisher']+['callNumLetters']+['callNumFull']+['physDesc']+['oclcDate'])
+f2=csv.writer(open(fileNameWithoutExtension+'oclcSearchNonMatches.csv', 'wb'))
+f2.writerow(['searchOoclcNum']+['borrower']+['lender']+['status']+['patronType']+['isbn']+['searchTitle']+['searchAuthor']+['searchDate'])
+with open(fileName) as csvfile:
+    reader = csv.DictReader(csvfile)
+    for row in reader:
+        borrower = row['BORROWER']
+        lender = row['LENDER']
+        status = row['STATUS']
+        patronType = row['PATRON TYPE']
+        searchOclcNum = row['OCLC']
+        print searchOclcNum
+        isbn = row['ISBN']
+        searchAuthor = row['AUTHOR']
+        searchTitle = row['TITLE']
+        searchPublisher = row['PUBLISHER']
+        searchDate = row['PUBLICATION YEAR']
+        try:
+            response = requests.get('http://www.worldcat.org/webservices/catalog/content/'+searchOclcNum+'?format=rss&wskey='+wskey).content
+            record = BeautifulSoup(response, "lxml").find('record')
+            oclcNum = record.find('controlfield', {'tag' : '001'}).text
+        except:
+            originalTitle = searchTitle
+            search = urllib.quote(searchTitle)
+            print search
+            response = requests.get(baseURL+search.strip()+'&count=1&format=rss&wskey='+wskey).content
+            record = BeautifulSoup(response, "lxml").findAll('item')
+            if record != []:
+                record = record[0]
+                url = record.find('guid').text.encode('utf-8')
+                oclcNum = url.replace('http://worldcat.org/oclc/','')
+                oclcAuthor = record.find('author').find('name').text.encode('utf-8')
+
+        response2 = requests.get(baseURL2+oclcNum+'?servicelevel=full&classificationScheme=LibraryOfCongress&wskey='+wskey).content
+        try:
+            record2 = BeautifulSoup(response2, "lxml").find('record')
+            try:
+                titleA = record2.find('datafield', {'tag' : '245'}).find('subfield', {'code' : 'a'}).text.encode('utf-8')
+            except:
+                titleA = ''
+            try:
+                titleB = record2.find('datafield', {'tag' : '245'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
+            except:
+                titleB = ''
+            oclcTitle = titleA + ' ' + titleB
+            oclcDate = record2.find('controlfield', {'tag' : '008'}).text[7:11].encode('utf-8')
+            try:
+                callNumFullA = record2.find('datafield', {'tag' : '050'}).find('subfield', {'code' : 'a'}).text.encode('utf-8')
+                numStart = re.search('\d', callNumFullA)
+                callNumLetters = callNumFullA[:numStart.start()]
+            except:
+                callNumFullA = ''
+                callNumLetters = ''
+            try:
+                callNumFullB = record2.find('datafield', {'tag' : '050'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
+            except:
+                callNumFullB = ''
+            callNumFull = callNumFullA + ' ' + callNumFullB
+            try:
+                oclcPublisher = record2.find('datafield', {'tag' : '260'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
+            except:
+                try:
+                    oclcPublisher = record2.find('datafield', {'tag' : '264'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
+                except:
+                    oclcPublisher = ''
+            try:
+                physDesc =  record2.find('datafield', {'tag' : '300'}).find('subfield', {'code' : 'a'}).text.encode('utf-8')
+            except:
+                physDesc = ''
+            f.writerow([searchOclcNum]+[borrower]+[lender]+[status]+[patronType]+[isbn]+[searchTitle]+[searchAuthor]+[searchDate]+[oclcNum]+[oclcTitle]+[oclcAuthor]+[oclcPublisher]+[callNumLetters]+[callNumFull]+[physDesc]+[oclcDate])
+            oclcNum = ''
+            oclcTitle = ''
+            oclcAuthor = ''
+            callNumLetters = ''
+            callNumFull = ''
+            oclcPublisher = ''
+            oclcDate = ''
+        except:
+            f2.writerow([searchOclcNum]+[borrower]+[lender]+[status]+[patronType]+[isbn]+[searchTitle]+[searchAuthor]+[searchDate])
+
+elapsedTime = time.time() - startTime
+m, s = divmod(elapsedTime, 60)
+h, m = divmod(m, 60)
+print 'Total script run time: ', '%d:%02d:%02d' % (h, m, s)

oclcTitlePhraseEnhanced.py

@@ -2,44 +2,63 @@
 from bs4 import BeautifulSoup
 import csv
 import secrets
+import urllib
 
 baseURL = 'http://www.worldcat.org/webservices/catalog/search/opensearch?q='
 baseURL2 = 'http://www.worldcat.org/webservices/catalog/content/'
 wskey = secrets.wskey
-f=csv.writer(open('resultsTitle.csv', 'wb'))
-f.writerow(['bibNumber']+['searchTitle']+['oclcTitle']+['url']+['author']+['publisher']+['encoding']+['lang']+['date'])
-with open('oclcRecordsTitle.txt') as txt:
-    for row in txt:
-        bibNumber = row[:row.index('|')]
-        searchTitle = row[row.index('|')+1:]
-        search = searchTitle.replace(' ','%20')
+f=csv.writer(open('oclcTitleSearchMatches.csv', 'wb'))
+f.writerow(['bibNumber']+['searchTitle']+['oclcTitle']+['oclcNum']+['url']+['author']+['publisher']+['physDesc']+['encoding']+['date'])
+f2=csv.writer(open('oclcTitleSearchNonMatches.csv', 'wb'))
+f2.writerow(['bibNumber']+['searchTitle'])
+with open('oclcRecordsTitle.csv') as csvfile:
+    reader = csv.DictReader(csvfile)
+    for row in reader:
+        bibNumber = row['bib#']
+        print bibNumber
+        searchTitle = row['245 - all subfields'][2:]
+        originalTitle = searchTitle
+        if 'b' in searchTitle:
+            searchTitle = searchTitle[:searchTitle.index('b')] + ' ' + searchTitle[searchTitle.index('b')+2:]
+            if 'c' in searchTitle:
+                searchTitle = searchTitle[:searchTitle.index('c')]
+            else:
+                pass
+        elif 'c' in searchTitle:
+            searchTitle = searchTitle[:searchTitle.index('c')]
+        else:
+            pass
+        search = urllib.quote(searchTitle)
         response = requests.get(baseURL+search.strip()+'&format=rss&wskey='+wskey).content
         records = BeautifulSoup(response, "lxml").findAll('item')
-        for record in records:
-            try:
-                title = record.find('title').text.encode('utf-8')
+        if records != []:
+            for record in records:
+                oclcTitle = record.find('title').text.encode('utf-8')
                 url = record.find('guid').text.encode('utf-8')
+                oclcNum = url.replace('http://worldcat.org/oclc/','')
                 author = record.find('author').find('name').text.encode('utf-8')
-            except:
-                title = ''
-                url = ''
-                author = ''
-            recordNumber = url.replace('http://worldcat.org/oclc/','')
-            response2 = requests.get(baseURL2+recordNumber+'?classificationScheme=LibraryOfCongress&wskey='+wskey).content
-            record2 = BeautifulSoup(response2, "lxml").find('record')
-            try:
+                response2 = requests.get(baseURL2+url+'?servicelevel=full&classificationScheme=LibraryOfCongress&wskey='+wskey).content
+                record2 = BeautifulSoup(response2, "lxml").find('record')
                 encoding = record2.find('leader').text[17].encode('utf-8')
+                type = record2.find('controlfield', {'tag' : '008'}).text[23:24]
                 date = record2.find('controlfield', {'tag' : '008'}).text[7:11].encode('utf-8')
-                lang = record2.find('controlfield', {'tag' : '008'}).text[35:38].encode('utf-8')
-            except:
-                date = ''
-                lang = ''
-                encoding = ''
-            try:
-                publisher = record2.find('datafield', {'tag' : '260'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
-            except:
                 try:
-                    publisher = record2.find('datafield', {'tag' : '264'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
+                    publisher = record2.find('datafield', {'tag' : '260'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
                 except:
-                    publisher = ''
-            f.writerow([bibNumber]+[searchTitle]+[title]+[url]+[author]+[publisher]+[encoding]+[lang]+[date])
+                    try:
+                        publisher = record2.find('datafield', {'tag' : '264'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
+                    except:
+                        publisher = ''
+                try:
+                    catLang =  record2.find('datafield', {'tag' : '040'}).find('subfield', {'code' : 'b'}).text.encode('utf-8')
+                except:
+                    catLang = ''
+                try:
+                    physDesc =  record2.find('datafield', {'tag' : '300'}).find('subfield', {'code' : 'a'}).text.encode('utf-8')
+                except:
+                    physDesc = ''
+                if type == ' ' and (catLang == 'eng' or catLang == ''):
+                    f.writerow([bibNumber]+[searchTitle]+[oclcTitle]+[oclcNum]+[url]+[author]+[publisher]+[physDesc]+[encoding]+[date])
+            f.writerow(['']+['']+['']+['']+['']+['']+['']+['']+['']+[''])
+        else:
+            f2.writerow([bibNumber]+[searchTitle])