diff --git a/Gemfile b/Gemfile index f22ae67716e..1fec0a0a12b 100644 --- a/Gemfile +++ b/Gemfile @@ -44,8 +44,6 @@ gem 'fog-aliyun' gem 'fog-aws' gem 'fog-core', '~> 2.6.0' gem 'fog-google', '~> 1.29.4' -gem 'fog-local' -gem 'fog-openstack' gem 'cf-uaa-lib', '~> 4.0.9' diff --git a/Gemfile.lock b/Gemfile.lock index da89b5ef77d..96769b8e52d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -125,11 +125,6 @@ GEM fog-json (1.3.0) fog-core multi_json (~> 1.10) - fog-local (0.9.0) - fog-core (>= 1.27, < 3.0) - fog-openstack (1.1.5) - fog-core (~> 2.1) - fog-json (>= 1.0) fog-xml (0.1.5) fog-core nokogiri (>= 1.5.11, < 2.0.0) @@ -553,8 +548,6 @@ DEPENDENCIES fog-aws fog-core (~> 2.6.0) fog-google (~> 1.29.4) - fog-local - fog-openstack googleapis-common-protos (>= 1.8.0) hashdiff httpclient diff --git a/lib/cloud_controller/blobstore/fog/fog_client.rb b/lib/cloud_controller/blobstore/fog/fog_client.rb index 217093f8130..04ff2f3fe8b 100644 --- a/lib/cloud_controller/blobstore/fog/fog_client.rb +++ b/lib/cloud_controller/blobstore/fog/fog_client.rb @@ -37,7 +37,7 @@ def initialize(connection_config:, end def local? - @connection_config[:provider].downcase == 'local' + false end def exists?(key) @@ -70,7 +70,7 @@ def cp_to_blobstore(source_path, destination_key) key: partitioned_key(destination_key), body: file, content_type: mime_type || 'application/zip', - public: local? + public: false }.merge(formatted_storage_options) files.create(options) @@ -119,19 +119,11 @@ def blob(key) end def files_for(prefix, _ignored_directory_prefixes=[]) - if connection.is_a? Fog::Local::Storage::Real - directory = connection.directories.get(File.join(dir.key, prefix || '')) - directory ? directory.files : [] - else - connection.directories.get(dir.key, prefix:).files - end + connection.directories.get(dir.key, prefix:).files end def ensure_bucket_exists - return if local? - options = { max_keys: 1 } - options['limit'] = 1 if connection.service == Fog::OpenStack::Storage connection.directories.get(@directory_key, options) || connection.directories.create(key: @directory_key, public: false) end @@ -205,7 +197,6 @@ def dir def connection options = @connection_config blobstore_timeout = options.delete(:blobstore_timeout) - options = options.merge(endpoint: '') if local? connection_options = options[:connection_options] || {} connection_options = connection_options.merge(read_timeout: blobstore_timeout, write_timeout: blobstore_timeout) options = options.merge(connection_options:) diff --git a/lib/cloud_controller/blobstore/fog/providers.rb b/lib/cloud_controller/blobstore/fog/providers.rb index 439de37e2d7..76def0703be 100644 --- a/lib/cloud_controller/blobstore/fog/providers.rb +++ b/lib/cloud_controller/blobstore/fog/providers.rb @@ -8,8 +8,6 @@ require 'fog/aliyun' require 'fog/aws' -require 'fog/local' require 'fog/google' -require 'fog/openstack' Fog::Logger[:deprecation] = original diff --git a/spec/api/documentation/buildpack_cache_api_spec.rb b/spec/api/documentation/buildpack_cache_api_spec.rb index c4b21119278..16ec1789c60 100644 --- a/spec/api/documentation/buildpack_cache_api_spec.rb +++ b/spec/api/documentation/buildpack_cache_api_spec.rb @@ -17,8 +17,9 @@ droplets: { droplet_directory_key: 'cc-droplets', fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('droplets', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' } }, directories: { @@ -31,10 +32,11 @@ before do TestConfig.override(**blobstore_config) + CloudController::DependencyLocator.instance.buildpack_cache_blobstore.ensure_bucket_exists end after do - Fog.mock! + Fog::Mock.reset FileUtils.rm_rf(workspace) end @@ -48,7 +50,6 @@ to delete unnecessary blobs. EOS - Fog.unmock! blobstore = CloudController::DependencyLocator.instance.buildpack_cache_blobstore blobstore.cp_to_blobstore(file, key) expect(blobstore).to exist(key) diff --git a/spec/support/bootstrap/test_config.rb b/spec/support/bootstrap/test_config.rb index 729cd775f0d..cb72ca18060 100644 --- a/spec/support/bootstrap/test_config.rb +++ b/spec/support/bootstrap/test_config.rb @@ -93,9 +93,10 @@ def configure_components(config) end def is_using_local_blobstore?(config) - res_pool_connection_provider = config.get(:resource_pool, :fog_connection)[:provider].downcase - packages_connection_provider = config.get(:packages, :fog_connection)[:provider].downcase - res_pool_connection_provider == 'local' || packages_connection_provider == 'local' + res_pool_blobstore_type = config.get(:resource_pool, :blobstore_type) + packages_blobstore_type = config.get(:packages, :blobstore_type) + + res_pool_blobstore_type&.start_with?('local') || packages_blobstore_type&.start_with?('local') end end end diff --git a/spec/unit/controllers/internal/download_droplets_controller_spec.rb b/spec/unit/controllers/internal/download_droplets_controller_spec.rb index 2150498c0d3..d581fc5e985 100644 --- a/spec/unit/controllers/internal/download_droplets_controller_spec.rb +++ b/spec/unit/controllers/internal/download_droplets_controller_spec.rb @@ -5,21 +5,22 @@ module VCAP::CloudController RSpec.describe DownloadDropletsController do describe 'GET /internal/v2/droplets/:guid/:droplet_hash/download' do - let(:workspace) { Dir.mktmpdir } let(:original_staging_config) do { packages: { fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('packages', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' }, app_package_directory_key: 'cc-packages' }, droplets: { droplet_directory_key: 'cc-droplets', fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('droplets', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' } } } @@ -34,11 +35,11 @@ module VCAP::CloudController let(:droplet) { DropletModel.make(state: 'STAGED') } before do - Fog.unmock! TestConfig.override(**staging_config) + blobstore.ensure_bucket_exists end - after { FileUtils.rm_rf(workspace) } + after { Fog::Mock.reset } def get_and_redirect(url) get url @@ -68,28 +69,6 @@ def upload_droplet end end - context 'when using with nginx' do - it 'succeeds for valid droplets' do - upload_droplet - - get_and_redirect "/internal/v2/droplets/#{process.guid}/#{process.droplet_checksum}/download" - expect(last_response.status).to eq(200) - expect(last_response.headers['X-Accel-Redirect']).to match("/cc-droplets/.*/#{process.droplet_hash}") - end - end - - context 'when not using with nginx' do - let(:staging_config) { original_staging_config.merge(nginx: { use_nginx: false }) } - - it 'succeeds for valid droplets' do - upload_droplet - - get_and_redirect "/internal/v2/droplets/#{process.guid}/#{process.droplet_checksum}/download" - expect(last_response.status).to eq(200) - expect(last_response.body).to eq('droplet contents') - end - end - context 'with a valid app but no droplet in the blobstore' do before do droplet.update(droplet_hash: 'not-in-blobstore') @@ -100,11 +79,6 @@ def upload_droplet expect(last_response.status).to eq(400) expect(decoded_response['description']).to eq("Staging error: droplet not found for #{process.guid}") end - - it 'fails if blobstore is remote' do - get_and_redirect "/internal/v2/droplets/#{process.guid}/#{process.droplet_checksum}/download" - expect(last_response.status).to eq(400) - end end context 'with an invalid droplet_hash' do @@ -123,21 +97,22 @@ def upload_droplet end describe 'GET /internal/v4/droplets/:guid/:droplet_hash/download' do - let(:workspace) { Dir.mktmpdir } let(:original_staging_config) do { packages: { fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('packages', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' }, app_package_directory_key: 'cc-packages' }, droplets: { droplet_directory_key: 'cc-droplets', fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('droplets', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' } } } @@ -152,11 +127,11 @@ def upload_droplet let(:droplet) { DropletModel.make(state: 'STAGED') } before do - Fog.unmock! TestConfig.override(**staging_config) + blobstore.ensure_bucket_exists end - after { FileUtils.rm_rf(workspace) } + after { Fog::Mock.reset } def upload_droplet(target_droplet=droplet) droplet_file = Tempfile.new(v3_app.guid) @@ -169,9 +144,12 @@ def upload_droplet(target_droplet=droplet) context 'when using with a revision' do let(:new_droplet) { DropletModel.make(state: 'STAGED') } - it 'succeeds when the the revisions droplet doesnt match the processes "desired" droplet' do + it 'redirects to the correct droplet when revision droplet differs from desired droplet' do upload_droplet upload_droplet(new_droplet) + allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).with(droplet).and_return('http://example.com/wrong/droplet') + allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).with(new_droplet).and_return('http://example.com/correct/droplet') + new_droplet.reload v3_app.update(revisions_enabled: true) @@ -179,31 +157,20 @@ def upload_droplet(target_droplet=droplet) process.update(revision:) get "/internal/v4/droplets/#{process.guid}/#{new_droplet.checksum}/download" - expect(last_response.status).to eq(200), last_response.body - expect(last_response.headers['X-Accel-Redirect']).to match("/cc-droplets/.*/#{new_droplet.droplet_hash}") - end - end - - context 'when using with nginx' do - it 'succeeds for valid droplets' do - upload_droplet - get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download" - expect(last_response.status).to eq(200) - expect(last_response.headers['X-Accel-Redirect']).to match("/cc-droplets/.*/#{process.droplet_hash}") + expect(last_response).to be_redirect + expect(last_response.headers['Location']).to eq('http://example.com/correct/droplet') end end - context 'when not using with nginx' do - let(:staging_config) { original_staging_config.merge(nginx: { use_nginx: false }) } + it 'redirects to the url provided by the blobstore_url_generator' do + upload_droplet + allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).and_return('http://example.com/somewhere/else') - it 'succeeds for valid droplets' do - upload_droplet + get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download" - get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download" - expect(last_response.status).to eq(200) - expect(last_response.body).to eq('droplet contents') - end + expect(last_response).to be_redirect + expect(last_response.headers['Location']).to eq('http://example.com/somewhere/else') end context 'with a valid app but no droplet in the blobstore' do @@ -216,12 +183,6 @@ def upload_droplet(target_droplet=droplet) expect(last_response.status).to eq(400) expect(decoded_response['description']).to eq("Staging error: droplet not found for #{process.guid}") end - - it 'fails if blobstore is remote' do - allow_any_instance_of(CloudController::Blobstore::Client).to receive(:local?).and_return(false) - get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download" - expect(last_response.status).to eq(400) - end end context 'with an invalid droplet_hash' do @@ -237,44 +198,6 @@ def upload_droplet(target_droplet=droplet) expect(last_response.status).to eq(404) end end - - context 'when the blobstore is not local' do - before do - allow_any_instance_of(CloudController::Blobstore::FogClient).to receive(:local?).and_return(false) - end - - it 'redirects to the url provided by the blobstore_url_generator' do - upload_droplet - allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).and_return('http://example.com/somewhere/else') - - get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download" - - expect(last_response).to be_redirect - expect(last_response.headers['Location']).to eq('http://example.com/somewhere/else') - end - - context 'when using with a revision' do - let(:new_droplet) { DropletModel.make(state: 'STAGED') } - - it 'succeeds when the the revisions droplet doesnt match the processes "desired" droplet' do - upload_droplet - upload_droplet(new_droplet) - allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).with(droplet).and_return('http://example.com/wrong/droplet') - allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).with(new_droplet).and_return('http://example.com/correct/droplet') - - new_droplet.reload - - v3_app.update(revisions_enabled: true) - revision = RevisionModel.make(app: v3_app, droplet: new_droplet) - process.update(revision:) - - get "/internal/v4/droplets/#{process.guid}/#{new_droplet.checksum}/download" - - expect(last_response).to be_redirect - expect(last_response.headers['Location']).to eq('http://example.com/correct/droplet') - end - end - end end end end diff --git a/spec/unit/controllers/runtime/stagings_controller_spec.rb b/spec/unit/controllers/runtime/stagings_controller_spec.rb index 2be7d38dc8c..ebb7671e07b 100644 --- a/spec/unit/controllers/runtime/stagings_controller_spec.rb +++ b/spec/unit/controllers/runtime/stagings_controller_spec.rb @@ -198,22 +198,25 @@ module VCAP::CloudController resource_pool: { resource_directory_key: 'cc-resources', fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('resourse_pool', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' } }, packages: { fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('packages', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' }, app_package_directory_key: 'cc-packages' }, droplets: { droplet_directory_key: 'cc-droplets', fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('droplets', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' } }, directories: { @@ -235,13 +238,15 @@ module VCAP::CloudController end before do - Fog.unmock! TestConfig.override(**staging_config) set_current_user_as_admin(user: User.make(guid: '1234'), email: 'joe@joe.com', user_name: 'briggs') allow_any_instance_of(BitsExpiration).to receive(:expire_packages!) end - after { FileUtils.rm_rf(workspace) } + after do + Fog::Mock.reset + FileUtils.rm_rf(workspace) + end shared_examples 'staging bad auth' do |verb, path| it 'returns 401 for bad credentials' do @@ -265,7 +270,6 @@ module VCAP::CloudController end describe 'GET /staging/packages/:guid' do - let(:package_without_bits) { PackageModel.make } let(:package) { PackageModel.make } before { authorize(staging_user, staging_password) } @@ -314,21 +318,10 @@ def create_test_blob end it 'fails if blobstore is not local' do - allow_any_instance_of(CloudController::Blobstore::FogClient).to receive(:local?).and_return(false) get '/staging/packages/some-guid' expect(last_response.status).to eq(400) end - it 'returns an error for non-existent packages' do - get '/staging/packages/bad-guid' - expect(last_response.status).to eq(404) - end - - it 'returns an error for an package without bits' do - get "/staging/packages/#{package_without_bits.guid}" - expect(last_response.status).to eq(404) - end - include_examples 'staging bad auth', :get, 'packages' end @@ -450,54 +443,14 @@ def create_test_blob describe 'GET /staging/v3/buildpack_cache/:stack/:app_guid/download' do let(:app_model) { AppModel.make } - let(:buildpack_cache) { Tempfile.new(app_model.guid) } let(:stack) { Sham.name } - before do - buildpack_cache.write('droplet contents') - buildpack_cache.close - - authorize staging_user, staging_password - end - - after { FileUtils.rm(buildpack_cache.path) } + before { authorize staging_user, staging_password } def make_request(guid=app_model.guid) get "/staging/v3/buildpack_cache/#{stack}/#{guid}/download" end - context 'with a valid buildpack cache' do - context 'when nginx is enabled' do - it 'redirects nginx to serve staged droplet' do - buildpack_cache_blobstore.cp_to_blobstore( - buildpack_cache.path, - "#{app_model.guid}/#{stack}" - ) - - make_request - expect(last_response.status).to eq(200) - expect(last_response.headers['X-Accel-Redirect']).to match("/cc-droplets/.*/#{app_model.guid}/#{stack}") - end - end - - context 'when nginx is disabled' do - let(:staging_config) do - original_staging_config.merge({ nginx: { use_nginx: false } }) - end - - it 'returns the buildpack cache' do - buildpack_cache_blobstore.cp_to_blobstore( - buildpack_cache.path, - "#{app_model.guid}/#{stack}" - ) - - make_request - expect(last_response.status).to eq(200) - expect(last_response.body).to eq('droplet contents') - end - end - end - context 'with a valid buildpack cache but no file' do it 'returns an error' do make_request @@ -518,57 +471,11 @@ def make_request(guid=app_model.guid) before { authorize(staging_user, staging_password) } - def upload_droplet - tmpdir = Dir.mktmpdir - zipname = File.join(tmpdir, 'test.zip') - TestZip.create(zipname, 10, 1024) - file_contents = File.read(zipname) - Jobs::V3::DropletUpload.new(zipname, droplet.guid, skip_state_transition: false).perform - FileUtils.rm_rf(tmpdir) - file_contents - end - - context 'when using with nginx' do - before { TestConfig.override(**staging_config) } - - it 'succeeds for valid droplets' do - upload_droplet - - get "/staging/v3/droplets/#{droplet.guid}/download" - expect(last_response.status).to eq(200) - - droplet.reload - expect(last_response.headers['X-Accel-Redirect']).to match("/cc-droplets/.*/#{droplet.blobstore_key}") - end - end - - context 'when not using with nginx' do - let(:staging_config) do - original_staging_config.merge({ nginx: { use_nginx: false } }) - end - - it 'succeeds for valid droplets' do - encoded_expected_body = Base64.encode64(upload_droplet) - - get "/staging/v3/droplets/#{droplet.guid}/download" - expect(last_response.status).to eq(200) - - encoded_actual_body = Base64.encode64(last_response.body) - expect(encoded_actual_body).to eq(encoded_expected_body) - end - end - it 'fails if blobstore is not local' do - allow_any_instance_of(CloudController::Blobstore::FogClient).to receive(:local?).and_return(false) get '/staging/v3/droplets/some-guid/download' expect(last_response.status).to eq(400) end - it 'returns an error for non-existent droplets' do - get '/staging/v3/droplets/bad-guid/download' - expect(last_response.status).to eq(404) - end - include_examples 'staging bad auth', :get, 'droplets' end end diff --git a/spec/unit/jobs/runtime/buildpack_cache_cleanup_spec.rb b/spec/unit/jobs/runtime/buildpack_cache_cleanup_spec.rb index e4c75cb56ef..9c5a7472a6c 100644 --- a/spec/unit/jobs/runtime/buildpack_cache_cleanup_spec.rb +++ b/spec/unit/jobs/runtime/buildpack_cache_cleanup_spec.rb @@ -18,8 +18,9 @@ module Jobs::Runtime droplets: { droplet_directory_key: 'cc-droplets', fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('droplets', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' } }, directories: { @@ -36,6 +37,8 @@ module Jobs::Runtime before do TestConfig.override(**blobstore_config) + blobstore.ensure_bucket_exists + droplet_blobstore.ensure_bucket_exists end let(:blobstore) do @@ -48,10 +51,10 @@ module Jobs::Runtime after do FileUtils.rm_rf(workspace) + Fog::Mock.reset end it 'deletes everything from the buildpack_cache directory' do - Fog.unmock! blobstore.cp_to_blobstore(file, orphan_key) blobstore.cp_to_blobstore(file, buildpack_cache_key) @@ -65,7 +68,6 @@ module Jobs::Runtime end it 'does not delete any droplets' do - Fog.unmock! blobstore.cp_to_blobstore(file, orphan_key) droplet_blobstore.cp_to_blobstore(file, droplet_key) diff --git a/spec/unit/jobs/v3/buildpack_cache_cleanup_spec.rb b/spec/unit/jobs/v3/buildpack_cache_cleanup_spec.rb index eabeb77fdc2..0f81ecb965e 100644 --- a/spec/unit/jobs/v3/buildpack_cache_cleanup_spec.rb +++ b/spec/unit/jobs/v3/buildpack_cache_cleanup_spec.rb @@ -18,8 +18,9 @@ module Jobs::V3 droplets: { droplet_directory_key: 'cc-droplets', fog_connection: { - provider: 'Local', - local_root: Dir.mktmpdir('droplets', workspace) + provider: 'AWS', + aws_access_key_id: 'fake', + aws_secret_access_key: 'fake' } }, directories: { @@ -36,6 +37,8 @@ module Jobs::V3 before do TestConfig.override(**blobstore_config) + blobstore.ensure_bucket_exists + droplet_blobstore.ensure_bucket_exists end let(:blobstore) do @@ -48,10 +51,10 @@ module Jobs::V3 after do FileUtils.rm_rf(workspace) + Fog::Mock.reset end it 'deletes everything from the buildpack_cache directory' do - Fog.unmock! blobstore.cp_to_blobstore(file, orphan_key) blobstore.cp_to_blobstore(file, buildpack_cache_key) @@ -65,7 +68,6 @@ module Jobs::V3 end it 'does not delete any droplets' do - Fog.unmock! blobstore.cp_to_blobstore(file, orphan_key) droplet_blobstore.cp_to_blobstore(file, droplet_key) diff --git a/spec/unit/jobs/v3/buildpack_cache_delete_spec.rb b/spec/unit/jobs/v3/buildpack_cache_delete_spec.rb index 3b51081a117..b556317fe62 100644 --- a/spec/unit/jobs/v3/buildpack_cache_delete_spec.rb +++ b/spec/unit/jobs/v3/buildpack_cache_delete_spec.rb @@ -5,9 +5,8 @@ module VCAP::CloudController module Jobs::V3 RSpec.describe BuildpackCacheDelete, job_context: :worker do let(:app_guid) { 'some-guid' } - let(:local_dir) { Dir.mktmpdir } let!(:blobstore) do - CloudController::Blobstore::FogClient.new(connection_config: { provider: 'Local', local_root: local_dir }, + CloudController::Blobstore::FogClient.new(connection_config: { provider: 'AWS', aws_access_key_id: 'fake', aws_secret_access_key: 'fake' }, directory_key: 'directory_key') end let(:path_1) { Presenters::V3::CacheKeyPresenter.cache_key(guid: app_guid, stack_name: 'stack1') } @@ -15,18 +14,17 @@ module Jobs::V3 let(:path_3) { Presenters::V3::CacheKeyPresenter.cache_key(guid: 'other-guid', stack_name: 'stack3') } before do - Fog.unmock! - path = File.join(local_dir, 'empty_file') - FileUtils.touch(path) - - allow(CloudController::DependencyLocator.instance).to receive(:buildpack_cache_blobstore).and_return(blobstore) - blobstore.cp_to_blobstore(path, path_1) - blobstore.cp_to_blobstore(path, path_2) - blobstore.cp_to_blobstore(path, path_3) + blobstore.ensure_bucket_exists + Tempfile.create('cache_file') do |f| + allow(CloudController::DependencyLocator.instance).to receive(:buildpack_cache_blobstore).and_return(blobstore) + blobstore.cp_to_blobstore(f.path, path_1) + blobstore.cp_to_blobstore(f.path, path_2) + blobstore.cp_to_blobstore(f.path, path_3) + end end after do - Fog.mock! + Fog::Mock.reset end subject(:job) { BuildpackCacheDelete.new(app_guid) } diff --git a/spec/unit/jobs/v3/droplet_bits_copier_spec.rb b/spec/unit/jobs/v3/droplet_bits_copier_spec.rb index f4e4ec0616b..21caf812716 100644 --- a/spec/unit/jobs/v3/droplet_bits_copier_spec.rb +++ b/spec/unit/jobs/v3/droplet_bits_copier_spec.rb @@ -6,21 +6,19 @@ module Jobs::V3 subject(:job) { DropletBitsCopier.new(source_droplet.guid, destination_droplet.guid) } let(:droplet_bits_path) { File.expand_path('../../../fixtures/good.zip', File.dirname(__FILE__)) } - let(:blobstore_dir) { Dir.mktmpdir } let(:droplet_blobstore) do - CloudController::Blobstore::FogClient.new(connection_config: { provider: 'Local', local_root: blobstore_dir }, + CloudController::Blobstore::FogClient.new(connection_config: { provider: 'AWS', aws_access_key_id: 'fake', aws_secret_access_key: 'fake' }, directory_key: 'droplet') end let(:source_droplet) { DropletModel.make(droplet_hash: 'abcdef1234', sha256_checksum: '4321fedcba', state: DropletModel::STAGED_STATE, app: nil) } let(:destination_droplet) { DropletModel.make(droplet_hash: nil, sha256_checksum: nil, state: DropletModel::STAGING_STATE, app: nil) } before do - Fog.unmock! + droplet_blobstore.ensure_bucket_exists end after do - Fog.mock! - FileUtils.remove_entry_secure blobstore_dir + Fog::Mock.reset end it { is_expected.to be_a_valid_job } diff --git a/spec/unit/jobs/v3/package_bits_copier_spec.rb b/spec/unit/jobs/v3/package_bits_copier_spec.rb index 46ff1d76dc1..3afda2f1c7f 100644 --- a/spec/unit/jobs/v3/package_bits_copier_spec.rb +++ b/spec/unit/jobs/v3/package_bits_copier_spec.rb @@ -6,21 +6,19 @@ module Jobs::V3 subject(:job) { PackageBitsCopier.new(source_package.guid, destination_package.guid) } let(:package_bits_path) { File.expand_path('../../../fixtures/good.zip', File.dirname(__FILE__)) } - let(:blobstore_dir) { Dir.mktmpdir } let(:package_blobstore) do - CloudController::Blobstore::FogClient.new(connection_config: { provider: 'Local', local_root: blobstore_dir }, + CloudController::Blobstore::FogClient.new(connection_config: { provider: 'AWS', aws_access_key_id: 'fake', aws_secret_access_key: 'fake' }, directory_key: 'package') end let(:source_package) { PackageModel.make(type: 'bits', package_hash: 'something', sha256_checksum: 'sha256') } let(:destination_package) { PackageModel.make(type: 'bits') } before do - Fog.unmock! + package_blobstore.ensure_bucket_exists end after do - Fog.mock! - FileUtils.remove_entry_secure blobstore_dir + Fog::Mock.reset end it { is_expected.to be_a_valid_job } diff --git a/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb b/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb index adf481da9a9..726b4cf78ec 100644 --- a/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb @@ -70,10 +70,6 @@ def upload_tmpfile(client, key='abcdef') allow(cdn).to receive(:download_uri).and_return(url_from_cdn) end - it 'is not local' do - expect(client).not_to be_local - end - it 'downloads through the CDN' do expect(cdn).to receive(:get). with('ab/cd/abcdef'). @@ -89,18 +85,6 @@ def upload_tmpfile(client, key='abcdef') end end - context 'a local blobstore' do - let(:connection_config) { { provider: 'Local', local_root: '/' } } - - before do - client.ensure_bucket_exists - end - - it 'is true if the provider is local' do - expect(client).to be_local - end - end - context 'common behaviors' do let(:directory) { Fog::Storage.new(connection_config).directories.create(key: directory_key) } let(:client) do @@ -303,16 +287,6 @@ def upload_tmpfile(client, key='abcdef') expect(client.blob(key).file.public_url).to be_nil end - it 'can copy as a public file' do - allow(client).to receive(:local?).and_return(true) - path = File.join(local_dir, 'empty_file') - FileUtils.touch(path) - key = 'abcdef12345' - - client.cp_to_blobstore(path, key) - expect(client.blob(key).file.public_url).to be - end - it 'sets content-type to mime-type of application/zip when not specified' do path = File.join(local_dir, 'empty_file') FileUtils.touch(path) @@ -541,14 +515,8 @@ def upload_tmpfile(client, key='abcdef') end describe '#delete_all' do - let(:connection_config) { { provider: 'Local', local_root: local_dir } } - before do - Fog.unmock! - end - - after do - Fog.mock! + client.ensure_bucket_exists end it 'deletes all the files' do @@ -585,7 +553,6 @@ def upload_tmpfile(client, key='abcdef') end it 'is ok if there are no files' do - Fog.mock! expect(directory.files).to have(0).items expect do client.delete_all @@ -593,7 +560,6 @@ def upload_tmpfile(client, key='abcdef') end it 'deletes in groups of the page_size' do - Fog.mock! connection = client.send(:connection) allow(connection).to receive(:delete_multiple_objects) @@ -647,14 +613,8 @@ def upload_tmpfile(client, key='abcdef') end describe '#delete_all_in_path' do - let(:connection_config) { { provider: 'Local', local_root: local_dir } } - before do - Fog.unmock! - end - - after do - Fog.mock! + client.ensure_bucket_exists end it 'deletes all the files within a specific path' do @@ -896,30 +856,27 @@ def wait_for_server_to_accept_requests(uri) end describe 'from a blobstore' do - let(:local_root) { File.expand_path('../../../../../', File.dirname(__FILE__)) } - let(:connection_config) { { provider: 'Local', local_root: local_root } } - let(:directory_key) { 'fixtures' } - - around do |example| - Fog.unmock! - example.run - Fog.mock! - end - it 'correctly downloads byte streams' do - Fog.unmock! - source_directory_path = File.join(local_root, directory_key) + content = 'some binary content for checksum verification' + source_file = Tempfile.new('source') + source_file.write(content) + source_file.close - source_file_path = File.join(source_directory_path, 'pa/rt/partitioned_key') - source_hexdigest = OpenSSL::Digest::SHA256.file(source_file_path).hexdigest + source_hexdigest = OpenSSL::Digest::SHA256.file(source_file.path).hexdigest - destination_file_path = File.join(Dir.mktmpdir, 'hard_file.xyz') + client.ensure_bucket_exists + client.cp_to_blobstore(source_file.path, 'partitioned_key') + destination_dir = Dir.mktmpdir + destination_file_path = File.join(destination_dir, 'hard_file.xyz') client.download_from_blobstore('partitioned_key', destination_file_path) destination_hexdigest = OpenSSL::Digest::SHA256.file(destination_file_path).hexdigest expect(destination_hexdigest).to eq(source_hexdigest) + ensure + source_file&.unlink + FileUtils.rm_rf(destination_dir) if destination_dir end end end diff --git a/spec/unit/lib/cloud_controller/packager/local_bits_packer_spec.rb b/spec/unit/lib/cloud_controller/packager/local_bits_packer_spec.rb index 1b0846a6e77..79667fe2c43 100644 --- a/spec/unit/lib/cloud_controller/packager/local_bits_packer_spec.rb +++ b/spec/unit/lib/cloud_controller/packager/local_bits_packer_spec.rb @@ -7,13 +7,12 @@ module CloudController::Packager let(:uploaded_files_path) { File.join(local_tmp_dir, 'good.zip') } let(:input_zip) { File.join(Paths::FIXTURES, 'good.zip') } - let(:blobstore_dir) { Dir.mktmpdir } let(:local_tmp_dir) { Dir.mktmpdir } let(:min_size) { 4 } let(:max_size) { 8 } let(:global_app_bits_cache) do CloudController::Blobstore::FogClient.new( - connection_config: { provider: 'Local', local_root: blobstore_dir }, + connection_config: { provider: 'AWS', aws_access_key_id: 'fake', aws_secret_access_key: 'fake' }, directory_key: 'global_app_bits_cache', min_size: min_size, max_size: max_size @@ -21,7 +20,7 @@ module CloudController::Packager end let(:package_blobstore) do CloudController::Blobstore::FogClient.new( - connection_config: { provider: 'Local', local_root: blobstore_dir }, + connection_config: { provider: 'AWS', aws_access_key_id: 'fake', aws_secret_access_key: 'fake' }, directory_key: 'package' ) end @@ -57,13 +56,13 @@ module CloudController::Packager nil end - Fog.unmock! + global_app_bits_cache.ensure_bucket_exists + package_blobstore.ensure_bucket_exists end after do - Fog.mock! + Fog::Mock.reset FileUtils.remove_entry_secure local_tmp_dir - FileUtils.remove_entry_secure blobstore_dir end describe '#send_package_to_blobstore' do