Skip to content

Remove fog-local and fog-openstack blobstore support #5197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
WeiQuan0605 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Remove fog-local and fog-openstack blobstore support #5197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2cb6fb9
Remove fog-local and fog-openstack blobstore support
WeiQuan0605 Jun 16, 2026
568022d
Fix missing local? method in FogClient
WeiQuan0605 Jun 16, 2026
9c286ff
Fix remaining test failures after fog-local/openstack removal
WeiQuan0605 Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Gemfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,6 @@ gem 'fog-aliyun'
gem 'fog-aws'
gem 'fog-core', '~> 2.6.0'
gem 'fog-google', '~> 1.29.4'
gem 'fog-local'
gem 'fog-openstack'

gem 'cf-uaa-lib', '~> 4.0.9'

Expand Down
7 changes: 0 additions & 7 deletions Gemfile.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,6 @@ GEM
fog-json (1.3.0)
fog-core
multi_json (~> 1.10)
fog-local (0.9.0)
fog-core (>= 1.27, < 3.0)
fog-openstack (1.1.5)
fog-core (~> 2.1)
fog-json (>= 1.0)
fog-xml (0.1.5)
fog-core
nokogiri (>= 1.5.11, < 2.0.0)
Expand Down Expand Up @@ -553,8 +548,6 @@ DEPENDENCIES
fog-aws
fog-core (~> 2.6.0)
fog-google (~> 1.29.4)
fog-local
fog-openstack
googleapis-common-protos (>= 1.8.0)
hashdiff
httpclient
Expand Down
15 changes: 3 additions & 12 deletions lib/cloud_controller/blobstore/fog/fog_client.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def initialize(connection_config:,
end

def local?
@connection_config[:provider].downcase == 'local'
false
end

def exists?(key)
Expand Down Expand Up @@ -70,7 +70,7 @@ def cp_to_blobstore(source_path, destination_key)
key: partitioned_key(destination_key),
body: file,
content_type: mime_type || 'application/zip',
public: local?
public: false
}.merge(formatted_storage_options)

files.create(options)
Expand Down Expand Up @@ -119,19 +119,11 @@ def blob(key)
end

def files_for(prefix, _ignored_directory_prefixes=[])
if connection.is_a? Fog::Local::Storage::Real
directory = connection.directories.get(File.join(dir.key, prefix || ''))
directory ? directory.files : []
else
connection.directories.get(dir.key, prefix:).files
end
connection.directories.get(dir.key, prefix:).files
end

def ensure_bucket_exists
return if local?

options = { max_keys: 1 }
options['limit'] = 1 if connection.service == Fog::OpenStack::Storage
connection.directories.get(@directory_key, options) || connection.directories.create(key: @directory_key, public: false)
end

Expand Down Expand Up @@ -205,7 +197,6 @@ def dir
def connection
options = @connection_config
blobstore_timeout = options.delete(:blobstore_timeout)
options = options.merge(endpoint: '') if local?
connection_options = options[:connection_options] || {}
connection_options = connection_options.merge(read_timeout: blobstore_timeout, write_timeout: blobstore_timeout)
options = options.merge(connection_options:)
Expand Down
2 changes: 0 additions & 2 deletions lib/cloud_controller/blobstore/fog/providers.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@

require 'fog/aliyun'
require 'fog/aws'
require 'fog/local'
require 'fog/google'
require 'fog/openstack'

Fog::Logger[:deprecation] = original
9 changes: 5 additions & 4 deletions spec/api/documentation/buildpack_cache_api_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@
droplets: {
droplet_directory_key: 'cc-droplets',
fog_connection: {
provider: 'Local',
local_root: Dir.mktmpdir('droplets', workspace)
provider: 'AWS',
aws_access_key_id: 'fake',
aws_secret_access_key: 'fake'
}
},
directories: {
Expand All @@ -31,10 +32,11 @@

before do
TestConfig.override(**blobstore_config)
CloudController::DependencyLocator.instance.buildpack_cache_blobstore.ensure_bucket_exists
end

after do
Fog.mock!
Fog::Mock.reset
FileUtils.rm_rf(workspace)
end

Expand All @@ -48,7 +50,6 @@
to delete unnecessary blobs.
EOS

Fog.unmock!
blobstore = CloudController::DependencyLocator.instance.buildpack_cache_blobstore
blobstore.cp_to_blobstore(file, key)
expect(blobstore).to exist(key)
Expand Down
7 changes: 4 additions & 3 deletions spec/support/bootstrap/test_config.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,10 @@ def configure_components(config)
end

def is_using_local_blobstore?(config)
res_pool_connection_provider = config.get(:resource_pool, :fog_connection)[:provider].downcase
packages_connection_provider = config.get(:packages, :fog_connection)[:provider].downcase
res_pool_connection_provider == 'local' || packages_connection_provider == 'local'
res_pool_blobstore_type = config.get(:resource_pool, :blobstore_type)
packages_blobstore_type = config.get(:packages, :blobstore_type)

res_pool_blobstore_type&.start_with?('local') || packages_blobstore_type&.start_with?('local')
end
end
end
133 changes: 28 additions & 105 deletions spec/unit/controllers/internal/download_droplets_controller_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,22 @@
module VCAP::CloudController
RSpec.describe DownloadDropletsController do
describe 'GET /internal/v2/droplets/:guid/:droplet_hash/download' do
let(:workspace) { Dir.mktmpdir }
let(:original_staging_config) do
{
packages: {
fog_connection: {
provider: 'Local',
local_root: Dir.mktmpdir('packages', workspace)
provider: 'AWS',
aws_access_key_id: 'fake',
aws_secret_access_key: 'fake'
},
app_package_directory_key: 'cc-packages'
},
droplets: {
droplet_directory_key: 'cc-droplets',
fog_connection: {
provider: 'Local',
local_root: Dir.mktmpdir('droplets', workspace)
provider: 'AWS',
aws_access_key_id: 'fake',
aws_secret_access_key: 'fake'
}
}
}
Expand All @@ -34,11 +35,11 @@ module VCAP::CloudController
let(:droplet) { DropletModel.make(state: 'STAGED') }

before do
Fog.unmock!
TestConfig.override(**staging_config)
blobstore.ensure_bucket_exists
end

after { FileUtils.rm_rf(workspace) }
after { Fog::Mock.reset }

def get_and_redirect(url)
get url
Expand Down Expand Up @@ -68,28 +69,6 @@ def upload_droplet
end
end

context 'when using with nginx' do
it 'succeeds for valid droplets' do
upload_droplet

get_and_redirect "/internal/v2/droplets/#{process.guid}/#{process.droplet_checksum}/download"
expect(last_response.status).to eq(200)
expect(last_response.headers['X-Accel-Redirect']).to match("/cc-droplets/.*/#{process.droplet_hash}")
end
end

context 'when not using with nginx' do
let(:staging_config) { original_staging_config.merge(nginx: { use_nginx: false }) }

it 'succeeds for valid droplets' do
upload_droplet

get_and_redirect "/internal/v2/droplets/#{process.guid}/#{process.droplet_checksum}/download"
expect(last_response.status).to eq(200)
expect(last_response.body).to eq('droplet contents')
end
end

context 'with a valid app but no droplet in the blobstore' do
before do
droplet.update(droplet_hash: 'not-in-blobstore')
Expand All @@ -100,11 +79,6 @@ def upload_droplet
expect(last_response.status).to eq(400)
expect(decoded_response['description']).to eq("Staging error: droplet not found for #{process.guid}")
end

it 'fails if blobstore is remote' do
get_and_redirect "/internal/v2/droplets/#{process.guid}/#{process.droplet_checksum}/download"
expect(last_response.status).to eq(400)
end
end

context 'with an invalid droplet_hash' do
Expand All @@ -123,21 +97,22 @@ def upload_droplet
end

describe 'GET /internal/v4/droplets/:guid/:droplet_hash/download' do
let(:workspace) { Dir.mktmpdir }
let(:original_staging_config) do
{
packages: {
fog_connection: {
provider: 'Local',
local_root: Dir.mktmpdir('packages', workspace)
provider: 'AWS',
aws_access_key_id: 'fake',
aws_secret_access_key: 'fake'
},
app_package_directory_key: 'cc-packages'
},
droplets: {
droplet_directory_key: 'cc-droplets',
fog_connection: {
provider: 'Local',
local_root: Dir.mktmpdir('droplets', workspace)
provider: 'AWS',
aws_access_key_id: 'fake',
aws_secret_access_key: 'fake'
}
}
}
Expand All @@ -152,11 +127,11 @@ def upload_droplet
let(:droplet) { DropletModel.make(state: 'STAGED') }

before do
Fog.unmock!
TestConfig.override(**staging_config)
blobstore.ensure_bucket_exists
end

after { FileUtils.rm_rf(workspace) }
after { Fog::Mock.reset }

def upload_droplet(target_droplet=droplet)
droplet_file = Tempfile.new(v3_app.guid)
Expand All @@ -169,41 +144,33 @@ def upload_droplet(target_droplet=droplet)
context 'when using with a revision' do
let(:new_droplet) { DropletModel.make(state: 'STAGED') }

it 'succeeds when the the revisions droplet doesnt match the processes "desired" droplet' do
it 'redirects to the correct droplet when revision droplet differs from desired droplet' do
upload_droplet
upload_droplet(new_droplet)
allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).with(droplet).and_return('http://example.com/wrong/droplet')
allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).with(new_droplet).and_return('http://example.com/correct/droplet')

new_droplet.reload

v3_app.update(revisions_enabled: true)
revision = RevisionModel.make(app: v3_app, droplet: new_droplet)
process.update(revision:)

get "/internal/v4/droplets/#{process.guid}/#{new_droplet.checksum}/download"
expect(last_response.status).to eq(200), last_response.body
expect(last_response.headers['X-Accel-Redirect']).to match("/cc-droplets/.*/#{new_droplet.droplet_hash}")
end
end

context 'when using with nginx' do
it 'succeeds for valid droplets' do
upload_droplet

get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download"
expect(last_response.status).to eq(200)
expect(last_response.headers['X-Accel-Redirect']).to match("/cc-droplets/.*/#{process.droplet_hash}")
expect(last_response).to be_redirect
expect(last_response.headers['Location']).to eq('http://example.com/correct/droplet')
end
end

context 'when not using with nginx' do
let(:staging_config) { original_staging_config.merge(nginx: { use_nginx: false }) }
it 'redirects to the url provided by the blobstore_url_generator' do
upload_droplet
allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).and_return('http://example.com/somewhere/else')

it 'succeeds for valid droplets' do
upload_droplet
get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download"

get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download"
expect(last_response.status).to eq(200)
expect(last_response.body).to eq('droplet contents')
end
expect(last_response).to be_redirect
expect(last_response.headers['Location']).to eq('http://example.com/somewhere/else')
end

context 'with a valid app but no droplet in the blobstore' do
Expand All @@ -216,12 +183,6 @@ def upload_droplet(target_droplet=droplet)
expect(last_response.status).to eq(400)
expect(decoded_response['description']).to eq("Staging error: droplet not found for #{process.guid}")
end

it 'fails if blobstore is remote' do
allow_any_instance_of(CloudController::Blobstore::Client).to receive(:local?).and_return(false)
get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download"
expect(last_response.status).to eq(400)
end
end

context 'with an invalid droplet_hash' do
Expand All @@ -237,44 +198,6 @@ def upload_droplet(target_droplet=droplet)
expect(last_response.status).to eq(404)
end
end

context 'when the blobstore is not local' do
before do
allow_any_instance_of(CloudController::Blobstore::FogClient).to receive(:local?).and_return(false)
end

it 'redirects to the url provided by the blobstore_url_generator' do
upload_droplet
allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).and_return('http://example.com/somewhere/else')

get "/internal/v4/droplets/#{process.guid}/#{process.droplet_checksum}/download"

expect(last_response).to be_redirect
expect(last_response.headers['Location']).to eq('http://example.com/somewhere/else')
end

context 'when using with a revision' do
let(:new_droplet) { DropletModel.make(state: 'STAGED') }

it 'succeeds when the the revisions droplet doesnt match the processes "desired" droplet' do
upload_droplet
upload_droplet(new_droplet)
allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).with(droplet).and_return('http://example.com/wrong/droplet')
allow_any_instance_of(CloudController::Blobstore::UrlGenerator).to receive(:droplet_download_url).with(new_droplet).and_return('http://example.com/correct/droplet')

new_droplet.reload

v3_app.update(revisions_enabled: true)
revision = RevisionModel.make(app: v3_app, droplet: new_droplet)
process.update(revision:)

get "/internal/v4/droplets/#{process.guid}/#{new_droplet.checksum}/download"

expect(last_response).to be_redirect
expect(last_response.headers['Location']).to eq('http://example.com/correct/droplet')
end
end
end
end
end
end
Loading
Loading