AP-483: Use the TIND API to retrieve records

Switches from the public `/search` endpoint to the actual `/api/v1/search` endpoint to retrieve metadata from TIND. Library consumers must set the environment variable LIT_TIND_API_KEY to a suitable TIND API token to authorize this endpoint.

At some point in the past week TIND restricted the ability of non-browser/human clients to search records, necessitating this change.

Author: jason-raitz

.github/workflows/build.yml

@@ -11,7 +11,7 @@ jobs:
 
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -24,7 +24,7 @@ jobs:
 
       - name: Upload artifacts
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: artifacts
+          name: artifacts-${{ matrix.os }}-${{ matrix.ruby }}
           path: artifacts/**

.github/workflows/gem-push.yml

@@ -13,7 +13,7 @@ jobs:
       packages: write
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1

CHANGES.md

@@ -1,3 +1,8 @@
+# 0.4.3 (2025-11-04)
+
+- BerkeleyLibrary::AV::Metadata::Readers:TIND now uses the /api/v1/search endpoint to retrieve records. The public /search endpoint stopped working for non-browser/human clients in the past week due to TIND changes.
+- BerkeleyLibrary::AV::Util#do_get sources the TIND API key from ENV['LIT_TIND_API_KEY'] if it's set.
+
 # 0.4.2 (2024-10-10)
 
 - Add TRANSCRIPTS to `AV::METADATA::FIELDS`
@@ -18,7 +23,7 @@
   - `Metadata#player_link_text`
   - `Metadata#player_url`
 - Remove the following constants:
-  - `AV::Constants::RESTRICTIONS_CALNET` 
+  - `AV::Constants::RESTRICTIONS_CALNET`
   - `AV::Constants::RESTRICTIONS_UCB_IP`
   - `AV::Constants::RESTRICTIONS`
   - `AV::Constants::RESTRICTIONS_NONE`

lib/berkeley_library/av/core/module_info.rb

@@ -8,7 +8,7 @@ class ModuleInfo
         SUMMARY = 'UC Berkeley Library audio/video core code'.freeze
         DESCRIPTION = 'Gem for UC Berkeley Library shared audio/video code'.freeze
         LICENSE = 'MIT'.freeze
-        VERSION = '0.4.2'.freeze
+        VERSION = '0.4.3'.freeze
         HOMEPAGE = 'https://github.com/BerkeleyLibrary/av-core'.freeze
 
         private_class_method :new

lib/berkeley_library/av/metadata/readers/tind.rb

@@ -22,9 +22,9 @@ def marc_uri_for(record_id)
             id_field = id_field_for(record_id)
             query_string = URI.encode_www_form(
               'p' => "#{id_field}:\"#{record_id}\"",
-              'of' => 'xm'
+              'format' => 'xml'
             )
-            URIs.append(base_uri, 'search', '?', query_string)
+            URIs.append(base_uri, 'api/v1/search', '?', query_string)
           end
 
           protected

lib/berkeley_library/av/util.rb

@@ -12,7 +12,11 @@ module Util
       DEFAULT_USER_AGENT = "#{Core::ModuleInfo::NAME} #{Core::ModuleInfo::VERSION} (#{Core::ModuleInfo::HOMEPAGE})".freeze
 
       def do_get(uri, ignore_errors: false)
-        body = URIs.get(uri, headers: { user_agent: DEFAULT_USER_AGENT })
+        headers = { user_agent: DEFAULT_USER_AGENT }
+        if uri.to_s.start_with?(BerkeleyLibrary::AV::Config.tind_base_uri.to_s) && ENV['LIT_TIND_API_KEY']
+          headers[:authorization] = "Token #{ENV['LIT_TIND_API_KEY']}"
+        end
+        body = URIs.get(uri, headers:)
         body && body.scrub
       rescue RestClient::Exception
         raise unless ignore_errors

spec/lib/berkeley_library/av/metadata/metadata_spec.rb

@@ -183,7 +183,7 @@ module AV
           it 'injects a TIND URL if not present (1/2)' do
             tind_035 = '(miscmat)00615'
             marc_xml = File.read("spec/data/record-#{tind_035}.xml")
-            search_url = "https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&of=xm"
+            search_url = "https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&format=xml"
             stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
             metadata = Metadata.for_record(record_id: tind_035)
 
@@ -200,7 +200,7 @@ module AV
           it 'injects a TIND URL if not present (2/2)' do
             tind_035 = 'physcolloquia-bk00169017b'
             marc_xml = File.read("spec/data/record-#{tind_035}.xml")
-            search_url = "https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&of=xm"
+            search_url = "https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&format=xml"
             stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
             metadata = Metadata.for_record(record_id: tind_035)
 
@@ -218,7 +218,7 @@ module AV
           it 'works for TIND records with OskiCat URLs' do
             tind_035 = '(pacradio)00107'
             marc_xml = File.read("spec/data/record-#{tind_035}.xml")
-            search_url = "https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&of=xm"
+            search_url = "https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&format=xml"
             stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
             metadata = Metadata.for_record(record_id: tind_035)
 
@@ -239,7 +239,7 @@ module AV
           it 'works for TIND-only records' do
             tind_035 = 'physcolloquia-bk00169017b'
             marc_xml = File.read("spec/data/record-#{tind_035}.xml")
-            search_url = "https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&of=xm"
+            search_url = "https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&format=xml"
             stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
             metadata = Metadata.for_record(record_id: tind_035)
 
@@ -259,7 +259,7 @@ module AV
           it 'returns the values' do
             tind_035 = 'physcolloquia-bk00169017b'
             marc_xml = File.read("spec/data/record-#{tind_035}.xml")
-            search_url = "https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&of=xm"
+            search_url = "https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&format=xml"
             stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
             metadata = Metadata.for_record(record_id: tind_035)
 

spec/lib/berkeley_library/av/metadata/source_spec.rb

@@ -161,7 +161,7 @@ class Metadata
 
           before do
             AV::Config.tind_base_uri = 'https://digicoll.lib.berkeley.edu'
-            @record_url = 'https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22%28pacradio%2900107%22&of=xm'
+            @record_url = 'https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22%28pacradio%2900107%22&format=xml'
           end
 
           after do

spec/lib/berkeley_library/av/record_spec.rb

@@ -46,7 +46,7 @@ module AV
         it 'returns a player URI based on the record ID for TIND records' do
           tind_035 = '(pacradio)01469'
           marc_xml = File.read("spec/data/record-#{tind_035}.xml")
-          search_url = "https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&of=xm"
+          search_url = "https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22#{CGI.escape(tind_035)}%22&format=xml"
           stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
 
           collection = 'Pacifica'
@@ -145,7 +145,7 @@ module AV
 
         it 'returns the TIND ID for TIND records' do
           marc_xml = File.read('spec/data/record-(pacradio)01469.xml')
-          search_url = 'https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22%28pacradio%2901469%22&of=xm'
+          search_url = 'https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22%28pacradio%2901469%22&format=xml'
           stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
 
           record = Record.from_metadata(
@@ -168,7 +168,7 @@ module AV
         it 'returns nil for TIND records with no bib number' do
           tind_id = '(clir)00020'
           marc_xml = File.read('spec/data/record-(clir)00020.xml')
-          search_url = 'https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22%28clir%2900020%22&of=xm'
+          search_url = 'https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22%28clir%2900020%22&format=xml'
           stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
 
           record = Record.from_metadata(collection: 'Video-Public-Bancroft', record_id: tind_id)
@@ -179,7 +179,7 @@ module AV
       describe :from_metadata do
         it 'loads the metadata' do
           marc_xml = File.read('spec/data/record-(pacradio)01469.xml')
-          search_url = 'https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22%28pacradio%2901469%22&of=xm'
+          search_url = 'https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22%28pacradio%2901469%22&format=xml'
           stub_request(:get, search_url).to_return(status: 200, body: marc_xml)
 
           record = Record.from_metadata(collection: 'Pacifica', record_id: '(pacradio)01469')
@@ -223,7 +223,7 @@ module AV
         end
 
         it "raises #{AV::RecordNotFound} if the record cannot be found" do
-          search_url = 'https://digicoll.lib.berkeley.edu/search?p=035__a%3A%22%28pacradio%2901469%22&of=xm'
+          search_url = 'https://digicoll.lib.berkeley.edu/api/v1/search?p=035__a%3A%22%28pacradio%2901469%22&format=xml'
           stub_request(:get, search_url).to_return(status: 404)
           expect do
             Record.from_metadata(

spec/lib/berkeley_library/av/util_spec.rb

@@ -24,6 +24,32 @@ module AV
           result = AV::Util.do_get(url, ignore_errors: true)
           expect(result).to be_nil
         end
+
+        it 'includes authorization header for TIND requests if LIT_TIND_API_KEY is set' do
+          expected_ua = Util::DEFAULT_USER_AGENT
+          expected_auth = 'Token some-long-api-token-value'
+
+          # expect do_get to call URIs.get with the correct headers
+          # but we don't need to actually perform the request
+          allow(BerkeleyLibrary::Util::URIs).to receive(:get).and_return('<response></response>')
+          allow(AV::Config).to receive(:tind_base_uri).and_return(URI('https://tind.example.edu/'))
+          url = "#{AV::Config.tind_base_uri}/some/api/endpoint"
+          ENV['LIT_TIND_API_KEY'] = 'some-long-api-token-value'
+          AV::Util.do_get(url)
+          expect(BerkeleyLibrary::Util::URIs).to have_received(:get).with(url, headers: { user_agent: expected_ua, authorization: expected_auth })
+        end
+
+        it 'does not include authorization header for non-TIND requests' do
+          expected_ua = Util::DEFAULT_USER_AGENT
+
+          # expect do_get to call URIs.get with the correct headers
+          # but we don't need to actually perform the request
+          allow(BerkeleyLibrary::Util::URIs).to receive(:get).and_return('<response></response>')
+          url = URI('https://alma.example.edu/some/api/endpoint')
+          ENV['LIT_TIND_API_KEY'] = nil
+          AV::Util.do_get(url)
+          expect(BerkeleyLibrary::Util::URIs).to have_received(:get).with(url, headers: { user_agent: expected_ua })
+        end
       end
     end
   end