Scratch asset importing

.env.example

@@ -62,3 +62,6 @@ SALESFORCE_CONNECT_PORT=5432
 SALESFORCE_CONNECT_DB=salesforce_development
 SALESFORCE_CONNECT_PASSWORD=password
 SALESFORCE_CONNECT_USER=postgres
+
+SCRATCH_ASSET_CONFIG_BASE_URL=https://example.com/config/
+SCRATCH_ASSET_IMPORT_BASE_URL=https://example.com/assets/

Gemfile

@@ -42,6 +42,7 @@ gem 'puma', '~> 7.2'
 gem 'rack_content_type_default', '~> 1.1'
 gem 'rack-cors'
 gem 'rails', '~> 7.1'
+gem 'ruby-progressbar', '~> 1.13', require: false
 gem 'sentry-rails'
 gem 'statesman'
 

Gemfile.lock

@@ -635,6 +635,7 @@ DEPENDENCIES
   rubocop-rspec_rails
   ruby-lsp
   ruby-lsp-rspec (~> 0.1.29)
+  ruby-progressbar (~> 1.13)
   selenium-webdriver
   sentry-rails
   shoulda-matchers (~> 7.0)

config/environments/development.rb

@@ -55,6 +55,8 @@
   # Print deprecation notices to the Rails logger.
   config.active_support.deprecation = :log
 
+  config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'debug')
+
   # Raise exceptions for disallowed deprecations.
   config.active_support.disallowed_deprecation = :raise
 

lib/corp_middleware.rb

@@ -12,7 +12,7 @@ def call(env)
     request_origin = env['HTTP_HOST']
     allowed_origins = OriginParser.parse_origins
 
-    if env['PATH_INFO'].start_with?('/rails/active_storage') && allowed_origins.any? do |origin|
+    if env['PATH_INFO'].start_with?('/rails/active_storage', '/api/scratch/assets/internalapi/asset/') && allowed_origins.any? do |origin|
          origin.is_a?(Regexp) ? origin =~ request_origin : origin == request_origin
        end
       headers['Cross-Origin-Resource-Policy'] = 'cross-origin'

lib/scratch_asset_importer.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'ruby-progressbar'
+require 'stringio'
+
+class ScratchAssetImporter
+  def self.import(...)
+    new(...).import
+  end
+
+  attr_reader :asset_base_url, :asset_names
+
+  ASSET_FETCHING_DELAY = 0.2
+
+  def initialize(asset_names, asset_base_url)
+    @asset_names = asset_names
+    @asset_base_url = asset_base_url
+  end
+
+  def import
+    bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: asset_names.count) if show_progress?
+
+    asset_names.each do |asset_name|
+      bar.increment if show_progress?
+      import_asset(asset_name)
+    end
+  end
+
+  private
+
+  def import_asset(asset_name)
+    return if ScratchAsset.exists?(filename: asset_name)
+
+    sleep(ASSET_FETCHING_DELAY)
+    asset = connection.get("#{asset_name}/get/")
+    ScratchAsset.create!(filename: asset_name).file.attach(io: StringIO.new(asset.body), filename: asset_name)
+  rescue StandardError => e
+    Rails.logger.error("Failed to import asset #{asset_name}: #{e.message}")
+  end
+
+  def connection
+    @connection ||= Faraday.new(url: asset_base_url) do |faraday|
+      faraday.response :raise_error
+    end
+  end
+
+  def show_progress?
+    !Rails.env.test?
+  end
+end

lib/scratch_config_importer.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class ScratchConfigImporter
+  def self.import(...)
+    new(...).import
+  end
+
+  attr_reader :asset_base_url, :asset_config_url
+
+  def initialize(asset_config_url, asset_base_url)
+    @asset_config_url = asset_config_url
+    @asset_base_url = asset_base_url
+  end
+
+  def import
+    config = connection.get.body
+    asset_config = JSON.parse(config, symbolize_names: true)
+    asset_names = extract_asset_names(asset_config)
+    ScratchAssetImporter.import(asset_names, asset_base_url)
+  end
+
+  def connection
+    Faraday.new(url: asset_config_url) do |faraday|
+      faraday.response :raise_error
+    end
+  end
+
+  private
+
+  def extract_asset_names(config)
+    names = []
+    config.each do |item|
+      names << item[:md5ext] if item[:md5ext]
+      names.concat(extract_asset_names(item.fetch(:costumes, [])))
+      names.concat(extract_asset_names(item.fetch(:sounds, [])))
+    end
+    names
+  end
+end

lib/tasks/scratch_assets.rake

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative 'seeds_helper'
+
+namespace :scratch_assets do
+  desc 'Import scratch assets'
+  task import_all: %i[import_backdrops import_costumes import_sounds import_sprites]
+
+  task import_backdrops: :environment do
+    Rails.logger.info 'Importing backdrops...'
+    config_url = "#{config_base_url}backdrops.json"
+    ScratchConfigImporter.import(config_url, import_base_url)
+  end
+
+  task import_costumes: :environment do
+    Rails.logger.info 'Importing costumes...'
+    config_url = "#{config_base_url}costumes.json"
+    ScratchConfigImporter.import(config_url, import_base_url)
+  end
+
+  task import_sounds: :environment do
+    Rails.logger.info 'Importing sounds...'
+    config_url = "#{config_base_url}sounds.json"
+    ScratchConfigImporter.import(config_url, import_base_url)
+  end
+
+  task import_sprites: :environment do
+    Rails.logger.info 'Importing sprites...'
+    config_url = "#{config_base_url}sprites.json"
+    ScratchConfigImporter.import(config_url, import_base_url)
+  end
+
+  def config_base_url
+    ENV.fetch('SCRATCH_ASSET_CONFIG_BASE_URL')
+  end
+
+  def import_base_url
+    ENV.fetch('SCRATCH_ASSET_IMPORT_BASE_URL')
+  end
+end

spec/lib/corp_middleware_spec.rb

@@ -18,6 +18,12 @@
     expect(headers['Cross-Origin-Resource-Policy']).to eq('cross-origin')
   end
 
+  it 'sets the Cross-Origin-Resource-Policy header for requests to scratch assets' do
+    _status, headers, _response = middleware.call(env.merge('PATH_INFO' => '/api/scratch/assets/internalapi/asset/123/get/'))
+
+    expect(headers['Cross-Origin-Resource-Policy']).to eq('cross-origin')
+  end
+
   it 'sets the Cross-Origin-Resource-Policy header for regex origin' do
     allow(ENV).to receive(:[]).with('ALLOWED_ORIGINS').and_return('/test\.com/')
 

spec/lib/scratch_asset_importer_spec.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'scratch_asset_importer'
+
+RSpec.describe ScratchAssetImporter do
+  describe '.import' do
+    it 'imports assets from the config' do
+      image = Rails.root.join('spec/fixtures/files/test_image_1.png').read
+      stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image)
+
+      described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/')
+
+      scratch_asset = ScratchAsset.find_by(filename: '123abc.png')
+      expect(scratch_asset).to be_present
+      expect(scratch_asset.file.download).to eq(image)
+    end
+
+    it 'does nothing if asset already exists' do
+      create(:scratch_asset, :with_file, filename: '123abc.png')
+
+      expect do
+        described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/')
+      end.not_to change(ScratchAsset, :count)
+    end
+
+    it 'can import multiple assets' do
+      image = Rails.root.join('spec/fixtures/files/test_image_1.png').read
+
+      stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image)
+      stub_request(:get, 'https://example.net/internalapi/asset/456xyz.png/get/').to_return(status: 200, body: image)
+
+      described_class.import(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
+      expect(ScratchAsset.find_by(filename: '123abc.png')).to be_present
+      expect(ScratchAsset.find_by(filename: '456xyz.png')).to be_present
+    end
+
+    it 'skips assets that fail to import' do
+      image = Rails.root.join('spec/fixtures/files/test_image_1.png').read
+
+      stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 500, body: 'error')
+      stub_request(:get, 'https://example.net/internalapi/asset/456xyz.png/get/').to_return(status: 200, body: image)
+
+      described_class.import(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
+      expect(ScratchAsset.find_by(filename: '123abc.png')).not_to be_present
+      expect(ScratchAsset.find_by(filename: '456xyz.png')).to be_present
+    end
+  end
+end

spec/lib/scratch_config_importer_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'scratch_asset_importer'
+
+RSpec.describe ScratchConfigImporter do
+  before do
+    allow(ScratchAssetImporter).to receive(:import)
+  end
+
+  describe '.import' do
+    it 'imports assets from the config' do
+      config = [{ md5ext: '123abc.png' }].to_json
+      stub_request(:get, 'https://example.com/config/backdrops.json').to_return(status: 200, body: config)
+
+      described_class.import('https://example.com/config/backdrops.json', 'https://example.net/internalapi/asset/')
+
+      expect(ScratchAssetImporter).to have_received(:import).with(['123abc.png'], 'https://example.net/internalapi/asset/')
+    end
+
+    it 'handles assets nested under sounds and costumes' do
+      config = [
+        {
+          costumes: [{
+            md5ext: '123abc.png'
+          }],
+          sounds: [{
+            md5ext: '456xyz.png'
+          }]
+        }
+      ].to_json
+      stub_request(:get, 'https://example.com/config/sprites.json').to_return(status: 200, body: config)
+
+      described_class.import('https://example.com/config/sprites.json', 'https://example.net/internalapi/asset/')
+
+      expect(ScratchAssetImporter).to have_received(:import).with(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
+    end
+  end
+end