From d122e78c0ed42cde8f1e4aaabb33b82c50016984 Mon Sep 17 00:00:00 2001 From: Vasily Fedoseyev Date: Mon, 8 Apr 2024 22:19:15 +0300 Subject: [PATCH] Remove cached storage --- lib/paperclip/storage.rb | 1 - lib/paperclip/storage/cached.rb | 226 -------------------------------- test/storage/cached_test.rb | 86 ------------ 3 files changed, 313 deletions(-) delete mode 100644 lib/paperclip/storage/cached.rb delete mode 100644 test/storage/cached_test.rb diff --git a/lib/paperclip/storage.rb b/lib/paperclip/storage.rb index fa3199b..060cef3 100644 --- a/lib/paperclip/storage.rb +++ b/lib/paperclip/storage.rb @@ -2,7 +2,6 @@ module Paperclip module Storage autoload :Filesystem, 'paperclip/storage/filesystem' autoload :DelayedUpload, 'paperclip/storage/delayed_upload' - autoload :Cached, 'paperclip/storage/cached' autoload :NoCacheS3, 'paperclip/storage/no_cache_s3' end end diff --git a/lib/paperclip/storage/cached.rb b/lib/paperclip/storage/cached.rb deleted file mode 100644 index 66bb8db..0000000 --- a/lib/paperclip/storage/cached.rb +++ /dev/null @@ -1,226 +0,0 @@ -module Paperclip - module Storage - # Saves file to `:cache` store, and run jobs to copy files to one ore more `:store` store. - # All stores are Fog::Storage::Directory instances (it has S3 and filesystem adapters). - # - # Options: - # - `:cache` - temporary storage, - # - `:stores` - one or more permanent storages (hash of {id => fog_directory}), - # first one is main others are mirrors, - # - `:key` - identifier template. - # - `:url` - hash of tamples {cache: t1, store: t2}. - # Values support :key interpolation which is merformed at configuration-time. - # - `:to_file_using_fog` - use fog interface in #to_file to fetch file from store. - # If disabled, downloads file by url via usual HTTP request. - # - # It uses `#{attachement_name}_synced_to_#{store_id}` field to mark that file - # is uploaded to particular storage. - module Cached - class << self - def included(base) - base.extend(ClassMethods) - end - end - - module ClassMethods - attr_reader :key_template, - :url_templates, - :directories, - :store_ids, - :main_store_id, - :download_by_url - - def setup(*) - super - - @key_template = options.fetch(:key) - @key_template = key_template[1..-1] if key_template.start_with?('/') - - @url_templates = options.fetch(:url).map { |k, v| [k, v.gsub(':key', key_template)] }.to_h - - @directories = options.fetch(:stores).symbolize_keys - @directories[:cache] = @options.fetch(:cache) - - @store_ids = options[:stores].keys.map(&:to_sym) - @main_store_id = store_ids.first - - @download_by_url = options[:download_by_url] - end - - def directory_for(store_id) - directories.fetch(store_id.to_sym) - end - - def synced_field_name(store_id) - @synced_field_names ||= store_ids.each_with_object({}) do |key, result| - result[key] = :"#{attachment_name}_synced_to_#{key}" - end - @synced_field_names[store_id.to_sym] - end - end - - def initialize(*) - super - @queued_jobs = [] - end - - def key(style = default_style) - interpolate(self.class.key_template, style) - end - - def storage_url(style = default_style) - current_store = synced_to?(self.class.main_store_id) ? :store : :cache - interpolate(self.class.url_templates.fetch(current_store), style) - end - - def reprocess! - super - flush_jobs - end - - # If store_id is given, it forces download from specific store using fog interface. - # Otherway it tries to download from cache store and finally uses url to download file - # via HTTP. This is the most compatible way to delayeds3. - def to_file(style = default_style, store_id = nil) - style_key = key(style) - return download_from_fog(store_id, style_key) if store_id - result = super(style) || download_from_fog(:cache, style_key) - return result if result - # Download by URL only if file is synced to main store. Similar to delayeds3. - return unless synced_to?(self.class.main_store_id) - if self.class.download_by_url - # FIXME: do we need to escape here? - uri = URI(URI::DEFAULT_PARSER.escape(storage_url(style))) - response = Net::HTTP.get_response(uri) - create_tempfile(response.body) if response.is_a?(Net::HTTPOK) - else - download_from_fog(self.class.main_store_id, style_key) - end - end - - def path(*) - raise '#path is not available for this type of storage, use #to_file instead' - end - - # Checks if attached file exists. When store_id is not given - # it uses fast check and does not perform API request for synced files - def exists?(style = default_style, store_id = nil) - return true if !store_id && synced_to?(self.class.main_store_id) - store_id ||= :cache - !self.class.directory_for(store_id).files.head(key(style)).nil? - end - - def flush_writes #:nodoc: - return if queued_for_write.empty? - write_to_directory(:cache, queued_for_write) - unless delay_processing? && dirty? - self.class.store_ids.each { |store_id| enqueue_sync_job(store_id) } - end - queued_for_write.clear - end - - # Important: It does not delete files from permanent stores. - def delete_styles_later(styles) - # если мы картинку заливали в облака, значит мы скорее всего ее уже удалили - # и можно не нагружать хранилище проверками - return if all_synced? - keys = styles.map { |x| key(x) } - -> { delete_keys_from(:cache, keys) } - end - - # Enqueues all pending jobs. First, jobs are placed to internal queue in flush_writes - # (in after_save) and this method pushes them for execution (in after_commit). - def flush_jobs - queued_jobs&.each(&:call)&.clear - end - - def upload_to(store_id) - sync_to(store_id) - clear_cache - end - - # Writes files from cache to permanent store. - def sync_to(store_id) - synced_field_name = self.class.synced_field_name(store_id) - return unless instance.respond_to?(synced_field_name) - return true if instance.public_send(synced_field_name) - files = self.class.all_styles.each_with_object({}) do |style, result| - file = to_file(style, :cache) - # For easier monitoring - unless file - raise "Missing cached files for #{instance.class.name}:#{instance.id}:#{style}" - end - result[style] = file - end - write_to_directory(store_id, files) - # ignore deleted objects and skip callbacks - if instance.class.unscoped.where(id: instance.id).update_all(synced_field_name => true) == 1 - instance.touch - instance[synced_field_name] = true - end - end - - def clear_cache - delete_styles_from(:cache) if all_synced? - end - - private - - def synced_to?(store_id) - instance.try(self.class.synced_field_name(store_id)) - end - - def all_synced? - self.class.store_ids.all? do |store_id| - synced_field_name = self.class.synced_field_name(store_id) - !instance.respond_to?(synced_field_name) || instance[synced_field_name] - end - end - - attr_reader :queued_jobs - - def enqueue_sync_job(store_id) - synced_field_name = self.class.synced_field_name(store_id) - return unless instance.respond_to?(synced_field_name) - instance.update_column(synced_field_name, false) if instance[synced_field_name] - queued_jobs.push -> { DelayedUpload.upload_later(self, store_id) } - end - - def download_from_fog(store_id, key) - body = self.class.directory_for(store_id).files.get(key)&.body - create_tempfile(body) if body - end - - def write_to_directory(store_id, files) - directory = self.class.directory_for(store_id) - common_options = { - content_type: instance_read(:content_type), - cache_control: "max-age=#{10.years.to_i}", - } - files.each do |style, file| - path = key(style) - log "Saving to #{store_id}:#{path}" - directory.files.create( - key: path, - public: true, - body: file, - **common_options, - ) - end - end - - def delete_styles_from(store_id, styles = self.class.all_styles) - keys = styles.map { |x| key(x) } - delete_keys_from(store_id, keys) - end - - def delete_keys_from(store_id, keys) - directory = self.class.directory_for(store_id) - keys.each do |key| - log("Deleting #{store_id}:#{key}") - directory.files.head(key)&.destroy - end - end - end - end -end diff --git a/test/storage/cached_test.rb b/test/storage/cached_test.rb deleted file mode 100644 index 75f621c..0000000 --- a/test/storage/cached_test.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -require 'test_helper' -require 'fog/local' -require 'sidekiq' -require 'sidekiq/testing' - -require 'delayed_paperclip' -DelayedPaperclip::Railtie.insert - -# rubocop:disable Naming/VariableNumber - -class FakeModel - attr_accessor :synced_to_store_1, :synced_to_store_2 -end - -class CachedStorageTest < Test::Unit::TestCase - TEST_ROOT = Pathname(__dir__).join('test') - - def fog_directory(suffix) - Fog::Storage.new(provider: 'Local', local_root: TEST_ROOT.join(suffix.to_s)) - .directories.new(key: '', public: true) - end - - def stub_file(name, content) - StringIO.new(content).tap { |string_io| string_io.stubs(:original_filename).returns(name) } - end - - setup do - rebuild_model( - storage: :cached, - key: ':filename', - url: { - cache: 'http://cache.local/:key', - store: 'http://store.local/:key' - }, - cache: fog_directory(:cache), - stores: { - store_1: fog_directory(:store_1), - store_2: fog_directory(:store_2) - } - ) - modify_table(:dummies) do |table| - table.boolean :avatar_synced_to_store_1, null: false, default: false - table.boolean :avatar_synced_to_store_2, null: false, default: false - end - @instance = Dummy.create - end - - teardown { TEST_ROOT.rmtree if TEST_ROOT.exist? } - - context 'assigning file' do - setup { Sidekiq::Testing.fake! } - - should 'write to cache and enqueue jobs' do - @instance.update!(avatar: stub_file('test.txt', 'qwe')) - @instance.reload - attachment = @instance.avatar - key = attachment.key - assert_equal true, attachment.exists? - assert_equal false, attachment.class.directory_for(:cache).files.head(key).nil? - assert_equal true, attachment.class.directory_for(:store_1).files.head(key).nil? - assert_equal true, attachment.class.directory_for(:store_2).files.head(key).nil? - assert_equal 'http://cache.local/test.txt', attachment.url(:original, false) - end - - context 'with inline jobs' do - setup { Sidekiq::Testing.inline! } - teardown { Sidekiq::Testing.fake! } - - should 'write to permanent stores and clear cache' do - @instance.update!(avatar: stub_file('test.txt', 'qwe')) - @instance.run_callbacks(:commit) - @instance.reload - attachment = @instance.avatar - key = attachment.key - assert_equal true, attachment.class.directory_for(:cache).files.head(key).nil? - assert_equal false, attachment.class.directory_for(:store_1).files.head(key).nil? - assert_equal false, attachment.class.directory_for(:store_2).files.head(key).nil? - assert_equal 'http://store.local/test.txt', attachment.url(:original, false) - end - end - end -end - -# rubocop:enable Naming/VariableNumber