From fd3c64d903d7f4ab527c899ed2843af6c88e98b0 Mon Sep 17 00:00:00 2001 From: Vasily Fedoseyev Date: Mon, 8 Apr 2024 22:09:27 +0300 Subject: [PATCH] =?UTF-8?q?=D0=92=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D1=8C=20=D0=BB=D0=BE=D0=BA=D0=B0=D0=BB=D1=8C?= =?UTF-8?q?=D0=BD=D0=BE=20=D0=BF=D0=BE=D1=81=D1=82=D0=B5=D1=81=D1=82=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D1=81=20minio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gemfile | 4 +- docker-compose.yml | 15 ++++++ lib/paperclip/storage/no_cache_s3.rb | 10 ++-- test/storage/no_cache_s3_test.rb | 78 +++++++++++++++++++++------- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/Gemfile b/Gemfile index 73e4987..f06c286 100644 --- a/Gemfile +++ b/Gemfile @@ -7,12 +7,12 @@ gemspec gem 'pg' -gem 'aws-sdk-s3' +gem 'aws-sdk-s3', '=1.143.0' gem 'fog-local' gem 'delayed_paperclip', github: 'insales/delayed_paperclip' gem 'rails' -gem 'sidekiq' +gem 'sidekiq', '~>6.5' # in 6.4.2 worker started to be renamed to job, in 7 removed gem 'test-unit' gem 'simplecov', require: false diff --git a/docker-compose.yml b/docker-compose.yml index aae4a88..4e88d7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,3 +10,18 @@ services: POSTGRES_HOST_AUTH_METHOD: trust ports: - 5432:5432 + + minio: + # image: bitnami/minio:2024.3.30 # fs backend removed, xl2 only + image: bitnami/minio:2022.10.29 + ports: + - '9002:9000' + - '9003:9001' + # volumes: + # - './tmp/minio:/bitnami/minio/data:rw' + # - './tmp/minio:/data:rw' + environment: + - MINIO_DEFAULT_BUCKETS=bucketname + - MINIO_ROOT_USER=test + - MINIO_ROOT_PASSWORD=testpassword + - MINIO_STORAGE_USE_HTTPS=false \ No newline at end of file diff --git a/lib/paperclip/storage/no_cache_s3.rb b/lib/paperclip/storage/no_cache_s3.rb index 20a71f4..8a0573d 100644 --- a/lib/paperclip/storage/no_cache_s3.rb +++ b/lib/paperclip/storage/no_cache_s3.rb @@ -170,13 +170,9 @@ module Paperclip return true if instance.public_send(synced_field_name) styles_to_upload = subject_to_post_process? ? self.class.all_styles : [:original] - files ||= styles_to_upload.each_with_object({}) do |style, result| - file = to_file(style, self.class.main_store_id) - # For easier monitoring - unless file - raise "Missing files in #{self.class.main_store_id} for #{instance.class.name}:#{instance.id}:#{style}" - end - result[style] = file + files ||= styles_to_upload.index_with do |style| + to_file(style, self.class.main_store_id) || + raise("Missing files in #{self.class.main_store_id} for #{instance.class.name}:#{instance.id}:#{style}") end write_to_store(store_id, files) # ignore deleted objects and skip callbacks diff --git a/test/storage/no_cache_s3_test.rb b/test/storage/no_cache_s3_test.rb index b942ea9..ded6962 100644 --- a/test/storage/no_cache_s3_test.rb +++ b/test/storage/no_cache_s3_test.rb @@ -11,10 +11,6 @@ DelayedPaperclip::Railtie.insert # rubocop:disable Naming/VariableNumber -class FakeModel - attr_accessor :synced_to_store_1, :synced_to_store_2 -end - class NoCacheS3Test < Test::Unit::TestCase TEST_ROOT = Pathname(__dir__).join('test') @@ -25,17 +21,25 @@ class NoCacheS3Test < Test::Unit::TestCase setup do rebuild_model( storage: :no_cache_s3, - key: ':filename', + key: "dummy_imgs/:id/:style-:filename", url: 'http://store.local/:key', stores: { store_1: { access_key_id: '123', secret_access_key: '123', region: 'r', bucket: 'buck' }, store_2: { access_key_id: '456', secret_access_key: '456', region: 'r', bucket: 'buck' } }, + # styles: { + # original: { geometry: '4x4>', processors: %i[thumbnail optimizer] }, # '4x4>' to limit size + # medium: '3x3', + # small: { geometry: '2x2', processors: [:recursive_thumbnail], thumbnail: :medium }, + # micro: { geometry: '1x1', processors: [:recursive_thumbnail], thumbnail: :small } + # } styles: { - original: { geometry: '4x4>', processors: %i[thumbnail optimizer] }, - medium: '3x3', - small: { geometry: '2x2', processors: [:recursive_thumbnail], thumbnail: :medium }, - micro: { geometry: '1x1', processors: [:recursive_thumbnail], thumbnail: :small } + original: { geometry: '2048x2048>', processors: %i[thumbnail optimizer] }, + large: '480x480', + medium: '240x240', + compact: { geometry: '160x160', processors: [:recursive_thumbnail], thumbnail: :medium }, + thumb: { geometry: '100x100', processors: [:recursive_thumbnail], thumbnail: :compact }, + micro: { geometry: '48x48', processors: [:recursive_thumbnail], thumbnail: :thumb } } ) modify_table(:dummies) do |table| @@ -48,15 +52,16 @@ class NoCacheS3Test < Test::Unit::TestCase @store1_stub.stubs(:url).returns('http://store.local') @store2_stub.stubs(:url).returns('http://store.local') @instance.avatar.class.stubs(:stores).returns({ store_1: @store1_stub, store_2: @store2_stub }) - Dummy::AvatarAttachment.any_instance.stubs(:to_file).returns( - stub_file('pixel.gif', Base64.decode64('R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw')) - ) + @gif_pixel = Base64.decode64('R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw') end teardown { TEST_ROOT.rmtree if TEST_ROOT.exist? } context 'assigning file' do - setup { Sidekiq::Testing.fake! } + setup do + Sidekiq::Testing.fake! + Dummy::AvatarAttachment.any_instance.stubs(:to_file).returns(stub_file('pixel.gif', @gif_pixel)) + end should 'set synced_fields to false' do @instance.avatar_synced_to_store_1 = true @@ -73,7 +78,7 @@ class NoCacheS3Test < Test::Unit::TestCase @instance.run_callbacks(:commit) @instance.reload attachment = @instance.avatar - assert_equal 'http://store.local/test.txt', attachment.url(:original, false) + assert_equal 'http://store.local/dummy_imgs/1/original-test.txt', attachment.url(:original, false) end context 'with inline jobs' do @@ -87,27 +92,60 @@ class NoCacheS3Test < Test::Unit::TestCase @instance.run_callbacks(:commit) @instance.reload attachment = @instance.avatar - assert_equal 'http://store.local/test.txt', attachment.url(:original, false) + assert_equal 'http://store.local/dummy_imgs/1/original-test.txt', attachment.url(:original, false) end end end + def assert_no_leftover_tmp + existing_files = Dir.children(Dir.tmpdir) + yield + leftover_files = (Dir.children(Dir.tmpdir) - existing_files).sort + assert_empty(leftover_files) + end + context "reprocess" do setup do Sidekiq::Testing.fake! - @instance.update_columns avatar_file_name: 'foo.gif', avatar_content_type: 'image/gif' + Dummy::AvatarAttachment.any_instance.stubs(:download_from_store).returns(stub_file('pixel.gif', @gif_pixel)) + @instance.update_columns avatar_file_name: 'foo.gif', avatar_content_type: 'image/gif', avatar_synced_to_store_1: true end should "delete tmp files" do @store1_stub.expects(:put_object).times(1 + (@instance.avatar.options[:styles].keys - [:original]).size) # Paperclip.expects(:log).with { puts "Log: #{_1}"; true }.at_least(3) - existing_files = Dir.children(Dir.tmpdir) - @instance.avatar.reprocess! - leftover_files = (Dir.children(Dir.tmpdir) - existing_files).sort - assert_empty(leftover_files) + assert_no_leftover_tmp { @instance.avatar.reprocess! } end end + context "with delayed_paperclip process_in_background" do + setup do + Dummy.process_in_background(:avatar) + Sidekiq::Testing.fake! + Sidekiq::Queues.clear_all + + # local minio + bucket = ::Aws::S3::Resource.new(client: ::Aws::S3::Client.new( + access_key_id: 'test', secret_access_key: 'testpassword', + endpoint: 'http://localhost:9002', region: 'laplandia', force_path_style: true + )).bucket("bucketname") + @instance.avatar.class.stubs(:stores).returns({ store_1: bucket }) + end + + should "add job and process" do + # @store1_stub.expects(:put_object).once + # @store2_stub.expects(:put_object).never + assert_no_leftover_tmp do + @instance.update!(avatar: stub_file('pixel.gif', @gif_pixel)) + # @instance.update!(avatar: File.open('sample_notebook_1.jpg')) + end + assert_equal(1, DelayedPaperclip::Jobs::Sidekiq.jobs.size) + + @instance = Dummy.find(@instance.id) + assert_no_leftover_tmp { DelayedPaperclip::Jobs::Sidekiq.perform_one } + end + end unless ENV['CI'] + context 'generating presigned_url' do setup do Dummy::AvatarAttachment.any_instance.stubs(:storage_url).returns('http://домен.pф/ключ?param1=параметр')