From 203ef7845007bee8379f9efc9b34033fe58bf026 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Sun, 4 Jul 2021 01:56:59 +0900 Subject: [PATCH 01/16] Add register_gem support for non-standard gemspecs --- .../bundle/create_bundle_build_file.rb | 55 ++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/ruby/private/bundle/create_bundle_build_file.rb b/ruby/private/bundle/create_bundle_build_file.rb index b283b40..a1baf7e 100755 --- a/ruby/private/bundle/create_bundle_build_file.rb +++ b/ruby/private/bundle/create_bundle_build_file.rb @@ -33,14 +33,14 @@ srcs = glob( include = [ ".bundle/config", - "{gem_lib_files}", - "lib/ruby/{ruby_version}/specifications/{name}-{version}.gemspec", + {gem_lib_files}, + "{gem_spec}", {gem_binaries} ], exclude = {exclude}, ), deps = {deps}, - includes = ["lib/ruby/{ruby_version}/gems/{name}-{version}/lib"], + includes = [{gem_lib_dirs}], ) GEM_TEMPLATE @@ -58,8 +58,22 @@ ) ALL_GEMS +# GEM_PATH = ->(ruby_version, gem_name, gem_version, platform) do +# platform_suffix = "-#{platform}" unless platform.nil? +# "lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}#{platform_suffix}" +# end + +# SPEC_PATH = ->(ruby_version, gem_name, gem_version, platform) do +# platform_suffix = "-#{platform}" unless platform.nil? +# "lib/ruby/#{ruby_version}/specifications/#{gem_name}-#{gem_version}#{platform_suffix}.gemspec" +# end + GEM_PATH = ->(ruby_version, gem_name, gem_version) do - "lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}" + Dir.glob("lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}*").first +end + +SPEC_PATH = ->(ruby_version, gem_name, gem_version) do + Dir.glob("lib/ruby/#{ruby_version}/specifications/#{gem_name}-#{gem_version}*.gemspec").first end require 'bundler' @@ -223,7 +237,33 @@ def remove_bundler_version! def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries) gem_path = GEM_PATH[ruby_version, spec.name, spec.version] - bundle_lib_paths << gem_lib_path = gem_path + '/lib' + spec_path = SPEC_PATH[ruby_version, spec.name, spec.version] + # puts "spec.source: #{spec.source}" + # puts "spec.source.class: #{spec.source.class}" # Bundler::Source::Rubygems + # puts "spec.platform: #{spec.platform}" # ruby + # puts "spec.source.specs: #{spec.source.specs.inspect}" # Bundler::Index + # puts "spec.source.specs.search(Gem::Platform.new(spec.platform)): #{spec.source.specs.search(Gem::Platform.new(spec.platform))}" # Bundler::Index + # pp spec + + # specification = spec.__materialize__ + # gem_lib_paths = specification.require_paths.map { |require_path| Path.join(gem_path, require_path) } + # bundle_lib_paths += gem_lib_paths + # pp spec + # + # gem_path = "lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}#{platform_suffix}" + puts "gem_path: #{gem_path}" + puts "spec_path: #{spec_path}" + + stub_spec = Gem::StubSpecification.gemspec_stub( + spec_path, + base_dir="lib/ruby/#{ruby_version}", + gems_dir="lib/ruby/#{ruby_version}/gems" + ) + pp spec + pp stub_spec + puts "stub_spec.require_paths: #{stub_spec.require_paths}" + gem_lib_dirs = stub_spec.require_paths.map { |require_path| File.join(gem_path, require_path) } + bundle_lib_paths += gem_lib_dirs # paths to search for executables gem_binaries = find_bundle_binaries(gem_path) @@ -234,8 +274,9 @@ def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries) warn("registering gem #{spec.name} with binaries: #{gem_binaries}") if bundle_binaries.key?(spec.name) template_out.puts GEM_TEMPLATE - .gsub('{gem_lib_path}', gem_lib_path) - .gsub('{gem_lib_files}', gem_lib_path + '/**/*') + .gsub('{gem_lib_files}', to_flat_string(gem_lib_dirs.map { |p| "#{p}/**/*" })) + .gsub('{gem_lib_dirs}', to_flat_string(gem_lib_dirs)) + .gsub('{gem_spec}', spec_path) .gsub('{gem_binaries}', to_flat_string(gem_binaries)) .gsub('{exclude}', exclude_array(spec.name).to_s) .gsub('{name}', spec.name) From 691595efff6efa88a43e4a8100c677643bfe8fc6 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Sun, 4 Jul 2021 12:55:49 +0900 Subject: [PATCH 02/16] cleanup and workaround --- .../bundle/create_bundle_build_file.rb | 43 ++++--------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/ruby/private/bundle/create_bundle_build_file.rb b/ruby/private/bundle/create_bundle_build_file.rb index a1baf7e..2389491 100755 --- a/ruby/private/bundle/create_bundle_build_file.rb +++ b/ruby/private/bundle/create_bundle_build_file.rb @@ -58,16 +58,6 @@ ) ALL_GEMS -# GEM_PATH = ->(ruby_version, gem_name, gem_version, platform) do -# platform_suffix = "-#{platform}" unless platform.nil? -# "lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}#{platform_suffix}" -# end - -# SPEC_PATH = ->(ruby_version, gem_name, gem_version, platform) do -# platform_suffix = "-#{platform}" unless platform.nil? -# "lib/ruby/#{ruby_version}/specifications/#{gem_name}-#{gem_version}#{platform_suffix}.gemspec" -# end - GEM_PATH = ->(ruby_version, gem_name, gem_version) do Dir.glob("lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}*").first end @@ -238,31 +228,14 @@ def remove_bundler_version! def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries) gem_path = GEM_PATH[ruby_version, spec.name, spec.version] spec_path = SPEC_PATH[ruby_version, spec.name, spec.version] - # puts "spec.source: #{spec.source}" - # puts "spec.source.class: #{spec.source.class}" # Bundler::Source::Rubygems - # puts "spec.platform: #{spec.platform}" # ruby - # puts "spec.source.specs: #{spec.source.specs.inspect}" # Bundler::Index - # puts "spec.source.specs.search(Gem::Platform.new(spec.platform)): #{spec.source.specs.search(Gem::Platform.new(spec.platform))}" # Bundler::Index - # pp spec - - # specification = spec.__materialize__ - # gem_lib_paths = specification.require_paths.map { |require_path| Path.join(gem_path, require_path) } - # bundle_lib_paths += gem_lib_paths - # pp spec - # - # gem_path = "lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}#{platform_suffix}" - puts "gem_path: #{gem_path}" - puts "spec_path: #{spec_path}" - - stub_spec = Gem::StubSpecification.gemspec_stub( - spec_path, - base_dir="lib/ruby/#{ruby_version}", - gems_dir="lib/ruby/#{ruby_version}/gems" - ) - pp spec - pp stub_spec - puts "stub_spec.require_paths: #{stub_spec.require_paths}" - gem_lib_dirs = stub_spec.require_paths.map { |require_path| File.join(gem_path, require_path) } + base_dir = "lib/ruby/#{ruby_version}" + + additional_requre_paths = Hash.new { |h, k| h[k] = [] } + additional_requre_paths['grpc'] = ['etc'] + # paths to register to $LOAD_PATH (usually ["lib"], but not always so.) + require_paths = Gem::StubSpecification.gemspec_stub(spec_path, base_dir, "#{base_dir}/gems").require_paths + require_paths += additional_requre_paths[spec.name] + gem_lib_dirs = require_paths.map { |require_path| File.join(gem_path, require_path) } bundle_lib_paths += gem_lib_dirs # paths to search for executables From 5a0e9b1b5c703a693f16523b83d3d3e73914bf53 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Sun, 4 Jul 2021 13:59:29 +0900 Subject: [PATCH 03/16] debug --- ruby/private/binary_wrapper.tpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ruby/private/binary_wrapper.tpl b/ruby/private/binary_wrapper.tpl index 09e0402..437ecf1 100755 --- a/ruby/private/binary_wrapper.tpl +++ b/ruby/private/binary_wrapper.tpl @@ -147,6 +147,10 @@ def main(args) system(gem_program + " pristine {gems_to_pristine}") end + puts "ruby_program: #{ruby_program}" + puts "rubyopt: #{rubyopt}" + puts "main: " + main + puts "args: #{args}" exec(ruby_program, *rubyopt, main, *args) # TODO(yugui) Support windows end From c2afe9be73294315079e890bef9dc2d9e2bb1a4d Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Sun, 4 Jul 2021 15:28:36 +0900 Subject: [PATCH 04/16] Add includes option to bundle_build --- ruby/private/binary_wrapper.tpl | 4 --- .../bundle/create_bundle_build_file.rb | 32 ++++++++++++------- ruby/private/bundle/def.bzl | 1 + ruby/private/constants.bzl | 3 ++ 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/ruby/private/binary_wrapper.tpl b/ruby/private/binary_wrapper.tpl index 437ecf1..09e0402 100755 --- a/ruby/private/binary_wrapper.tpl +++ b/ruby/private/binary_wrapper.tpl @@ -147,10 +147,6 @@ def main(args) system(gem_program + " pristine {gems_to_pristine}") end - puts "ruby_program: #{ruby_program}" - puts "rubyopt: #{rubyopt}" - puts "main: " + main - puts "args: #{args}" exec(ruby_program, *rubyopt, main, *args) # TODO(yugui) Support windows end diff --git a/ruby/private/bundle/create_bundle_build_file.rb b/ruby/private/bundle/create_bundle_build_file.rb index 2389491..f8f721c 100755 --- a/ruby/private/bundle/create_bundle_build_file.rb +++ b/ruby/private/bundle/create_bundle_build_file.rb @@ -33,7 +33,7 @@ srcs = glob( include = [ ".bundle/config", - {gem_lib_files}, + {gem_lib_dirs}, "{gem_spec}", {gem_binaries} ], @@ -148,6 +148,7 @@ class BundleBuildFileGenerator :repo_name, :build_file, :gemfile_lock, + :includes, :excludes, :ruby_version @@ -159,11 +160,14 @@ def initialize(workspace_name:, repo_name:, build_file: 'BUILD.bazel', gemfile_lock: 'Gemfile.lock', - excludes: nil) + includes: nil, + excludes: nil, + additional_require_paths: nil) @workspace_name = workspace_name @repo_name = repo_name @build_file = build_file @gemfile_lock = gemfile_lock + @includes = includes @excludes = excludes # This attribute returns 0 as the third minor version number, which happens to be # what Ruby uses in the PATH to gems, eg. ruby 2.6.5 would have a folder called @@ -230,11 +234,11 @@ def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries) spec_path = SPEC_PATH[ruby_version, spec.name, spec.version] base_dir = "lib/ruby/#{ruby_version}" - additional_requre_paths = Hash.new { |h, k| h[k] = [] } - additional_requre_paths['grpc'] = ['etc'] - # paths to register to $LOAD_PATH (usually ["lib"], but not always so.) + # paths to register to $LOAD_PATH + # Usually, registering the directory paths listed in the `require_paths` of gemspecs is sufficient, but + # some rubygems also require additional paths to be included in the load paths. require_paths = Gem::StubSpecification.gemspec_stub(spec_path, base_dir, "#{base_dir}/gems").require_paths - require_paths += additional_requre_paths[spec.name] + require_paths += include_array(spec.name) gem_lib_dirs = require_paths.map { |require_path| File.join(gem_path, require_path) } bundle_lib_paths += gem_lib_dirs @@ -246,8 +250,7 @@ def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries) warn("registering gem #{spec.name} with binaries: #{gem_binaries}") if bundle_binaries.key?(spec.name) - template_out.puts GEM_TEMPLATE - .gsub('{gem_lib_files}', to_flat_string(gem_lib_dirs.map { |p| "#{p}/**/*" })) + template_out.puts GEM_TEMPLATE#.gsub('{gem_lib_files}', to_flat_string(gem_lib_dirs.map { |p| "#{p}/**/*" })) .gsub('{gem_lib_dirs}', to_flat_string(gem_lib_dirs)) .gsub('{gem_spec}', spec_path) .gsub('{gem_binaries}', to_flat_string(gem_binaries)) @@ -276,6 +279,10 @@ def find_bundle_binaries(gem_path) .map { |binary| 'bin/' + binary } end + def include_array(gem_name) + (includes[gem_name] || []) + end + def exclude_array(gem_name) (excludes[gem_name] || []) + DEFAULT_EXCLUDES end @@ -285,18 +292,19 @@ def to_flat_string(array) end end -# ruby ./create_bundle_build_file.rb "BUILD.bazel" "Gemfile.lock" "repo_name" "[]" "wsp_name" +# ruby ./create_bundle_build_file.rb "BUILD.bazel" "Gemfile.lock" "repo_name" "{}" "{}" "wsp_name" if $0 == __FILE__ - if ARGV.length != 5 - warn("USAGE: #{$0} BUILD.bazel Gemfile.lock repo-name [excludes-json] workspace-name".orange) + if ARGV.length != 6 + warn("USAGE: #{$0} BUILD.bazel Gemfile.lock repo-name {includes-json} {excludes-json} workspace-name".orange) exit(1) end - build_file, gemfile_lock, repo_name, excludes, workspace_name, * = *ARGV + build_file, gemfile_lock, repo_name, includes, excludes, workspace_name, * = *ARGV BundleBuildFileGenerator.new(build_file: build_file, gemfile_lock: gemfile_lock, repo_name: repo_name, + includes: JSON.parse(includes), excludes: JSON.parse(excludes), workspace_name: workspace_name).generate! diff --git a/ruby/private/bundle/def.bzl b/ruby/private/bundle/def.bzl index fade53e..11928fa 100644 --- a/ruby/private/bundle/def.bzl +++ b/ruby/private/bundle/def.bzl @@ -144,6 +144,7 @@ def generate_bundle_build_file(runtime_ctx, previous_result): "BUILD.bazel", # Bazel build file (can be empty) "Gemfile.lock", # Gemfile.lock where we list all direct and transitive dependencies runtime_ctx.ctx.name, # Name of the target + repr(runtime_ctx.ctx.attr.includes), repr(runtime_ctx.ctx.attr.excludes), RULES_RUBY_WORKSPACE_NAME, ] diff --git a/ruby/private/constants.bzl b/ruby/private/constants.bzl index 633878f..0092584 100644 --- a/ruby/private/constants.bzl +++ b/ruby/private/constants.bzl @@ -82,6 +82,9 @@ BUNDLE_ATTRS = { "bundler_version": attr.string( default = DEFAULT_BUNDLER_VERSION, ), + "includes": attr.string_list_dict( + doc = "List of glob patterns per gem to be additionally loaded from the library", + ), "excludes": attr.string_list_dict( doc = "List of glob patterns per gem to be excluded from the library", ), From 0b048c54881bacfe67e7af1697196a10e79160f1 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Sun, 4 Jul 2021 22:25:50 +0900 Subject: [PATCH 05/16] Add description about includes/excludes optional parameters for ruby_bundle --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7cdb064..c6309de 100644 --- a/README.md +++ b/README.md @@ -551,7 +551,8 @@ ruby_bundle( gemfile, gemfile_lock, bundler_version = "2.1.2", - excludes = [], + includes = {}, + excludes = {}, ruby_sdk = "@org_ruby_lang_ruby_toolchain", ruby_interpreter = "@org_ruby_lang_ruby_toolchain//:ruby", ) @@ -600,6 +601,29 @@ ruby_bundle(

NOTE: This rule never updates the Gemfile.lock. It is your responsibility to generate/update Gemfile.lock

+ + includes + + Dictionary of key-value-pairs (key: string, value: list of strings), optional +

+ List of glob patterns per gem to be additionally loaded from the library. + Keys are the names of the gems which require some file/directory paths not listed in the require_paths attribute of the gemspecs to be also added to $LOAD_PATH at runtime. + Values are lists of blob path patterns. The path blob patterns are relative to the root directory of the gems. +

+ + + + excludes + + Dictionary of key-value-pairs (key: string, value: list of strings), optional +

+ List of glob patterns per gem to be excluded from the library. + Keys are the names of the gems. + Values are lists of blob path patterns. The path blob patterns are relative to the root directory of the gems. + The default value is ["**/* *.*", "**/* */*"] +

+ + From cde0144fd5f6a7297989dffda21f2d9e0cb2a8fd Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Sun, 4 Jul 2021 22:27:32 +0900 Subject: [PATCH 06/16] Trim extra whitespace characters --- README.md | 168 +++++++++++++++++++++++++++--------------------------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index c6309de..c84117a 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ Use `ruby_gem` rule to package any number of ruby files or folders into a Ruby-G ```bazel load( "@bazelruby_rules_ruby//ruby:defs.bzl", - "ruby_gem", + "ruby_gem", ) ruby_gem( @@ -201,7 +201,7 @@ ruby_gem( gem_version = "0.1.0", gem_summary = "Example gem to demonstrate Bazel Gem packaging", gem_description = "Example gem to demonstrate Bazel Gem packaging", - gem_homepage = "https://github.com/bazelruby/rules_ruby", + gem_homepage = "https://github.com/bazelruby/rules_ruby", gem_authors = [ "BazelRuby", "Konstantin Gredeskoul" @@ -219,7 +219,7 @@ ruby_gem( "rubocop": "", }, srcs = [ - glob("{bin,exe,lib,spec}/**/*.rb") + glob("{bin,exe,lib,spec}/**/*.rb") ], deps = [ "//lib:example_gem", @@ -238,7 +238,7 @@ If you are using ASDF to manage your ruby installs, you can use them by adding ` build --test_env=ASDF_DIR --test_env=ASDF_DATA_DIR build --action_env=ASDF_DIR --test_env=ASDF_DATA_DIR ``` -You will have to be sure to export the `ASDF_DATA_DIR` in your profile since it's not set by default. e.g. `export ASDF_DATA_DIR="$HOME/.asdf"` +You will have to be sure to export the `ASDF_DATA_DIR` in your profile since it's not set by default. e.g. `export ASDF_DATA_DIR="$HOME/.asdf"` ### Rule Dependency Diagram @@ -254,19 +254,19 @@ The following diagram attempts to capture the implementation behind `ruby_librar ```bazel ruby_library( - name, - deps, - srcs, - data, - compatible_with, - deprecation, - distribs, - features, - licenses, - restricted_to, - tags, - testonly, - toolchains, + name, + deps, + srcs, + data, + compatible_with, + deprecation, + distribs, + features, + licenses, + restricted_to, + tags, + testonly, + toolchains, visibility) ``` @@ -343,22 +343,22 @@ ruby_library( ```bazel ruby_binary( - name, - deps, - srcs, + name, + deps, + srcs, data, - main, - compatible_with, - deprecation, - distribs, - features, - licenses, - restricted_to, - tags, - testonly, - toolchains, - visibility, - args, + main, + compatible_with, + deprecation, + distribs, + features, + licenses, + restricted_to, + tags, + testonly, + toolchains, + visibility, + args, output_licenses ) ``` @@ -442,25 +442,25 @@ ruby_binary( ```bazel ruby_test( - name, - deps, - srcs, - data, - main, - compatible_with, - deprecation, - distribs, - features, - licenses, - restricted_to, - tags, - testonly, - toolchains, - visibility, - args, - size, - timeout, - flaky, + name, + deps, + srcs, + data, + main, + compatible_with, + deprecation, + distribs, + features, + licenses, + restricted_to, + tags, + testonly, + toolchains, + visibility, + args, + size, + timeout, + flaky, local, shard_count) ``` @@ -547,9 +547,9 @@ This rule installs gems defined in a Gemfile using Bundler, and exports individu ```bazel ruby_bundle( - name, - gemfile, - gemfile_lock, + name, + gemfile, + gemfile_lock, bundler_version = "2.1.2", includes = {}, excludes = {}, @@ -679,28 +679,28 @@ ruby_binary( ```bazel ruby_rspec( - name, - deps, - srcs, - data, - main, - rspec_args, - bundle, - compatible_with, - deprecation, - distribs, - features, - licenses, - restricted_to, - tags, - testonly, - toolchains, - visibility, - args, - size, - timeout, - flaky, - local, + name, + deps, + srcs, + data, + main, + rspec_args, + bundle, + compatible_with, + deprecation, + distribs, + features, + licenses, + restricted_to, + tags, + testonly, + toolchains, + visibility, + args, + size, + timeout, + flaky, + local, shard_count ) ``` @@ -810,7 +810,7 @@ ruby_gem( data = data ) ``` - + @@ -858,17 +858,17 @@ ruby_gem( - + - + - + - - + +
gem_description - String, required + String, required

Single-line, paragraph-sized description text for the gem.
gem_homepage - String, optional + String, optional

Homepage URL of the gem.

gem_authors @@ -887,7 +887,7 @@ ruby_gem( List of email addresses of the authors.

srcs @@ -918,7 +918,7 @@ ruby_gem( Typically this value is just `lib` (which is also the default).

gem_runtime_dependencies @@ -939,8 +939,8 @@ ruby_gem( testing gems, linters, code coverage and more.

From 00934b8a54464e66bf922c2c55eb7f2e68c98d3b Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Sun, 4 Jul 2021 22:39:58 +0900 Subject: [PATCH 07/16] Add example usage of ruby_bundle.includes to README --- README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c84117a..689712e 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,18 @@ load( ruby_bundle( name = "bundle", + # Specify additional paths to be loaded from the gems at runtime, if any. + includes = { + "grpc": [ + "etc/*", + "src/ruby/lib/*", + ], + "google-protobuf": [ + "lib/google/*", + "lib/google/2.7/*", + "lib/google/protobuf/**/*", + ] + }, excludes = { "mini_portile": ["test/**/*"], }, @@ -608,7 +620,7 @@ ruby_bundle(

List of glob patterns per gem to be additionally loaded from the library. Keys are the names of the gems which require some file/directory paths not listed in the require_paths attribute of the gemspecs to be also added to $LOAD_PATH at runtime. - Values are lists of blob path patterns. The path blob patterns are relative to the root directory of the gems. + Values are lists of blob path patterns, which are relative to the root directories of the gems.

@@ -619,7 +631,7 @@ ruby_bundle(

List of glob patterns per gem to be excluded from the library. Keys are the names of the gems. - Values are lists of blob path patterns. The path blob patterns are relative to the root directory of the gems. + Values are lists of blob path patterns, which are relative to the root directories of the gems. The default value is ["**/* *.*", "**/* */*"]

From f436147f48f8d53a7a1dbab9fca717daf583f013 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Mon, 5 Jul 2021 05:56:07 +0900 Subject: [PATCH 08/16] Add a test for includes option of ruby_bundle rule --- .../bundle_includes_workspace/BUILD.bazel | 15 ++++++++ .../bundle_includes_workspace/Gemfile | 5 +++ .../bundle_includes_workspace/Gemfile.lock | 13 +++++++ .../bundle_includes_workspace/WORKSPACE | 35 +++++++++++++++++++ .../bundle_includes_workspace/script.rb | 25 +++++++++++++ 5 files changed, 93 insertions(+) create mode 100644 ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel create mode 100644 ruby/tests/testdata/bundle_includes_workspace/Gemfile create mode 100644 ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock create mode 100644 ruby/tests/testdata/bundle_includes_workspace/WORKSPACE create mode 100644 ruby/tests/testdata/bundle_includes_workspace/script.rb diff --git a/ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel b/ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel new file mode 100644 index 0000000..4605a35 --- /dev/null +++ b/ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel @@ -0,0 +1,15 @@ +load( + "@bazelruby_rules_ruby//ruby:defs.bzl", + "ruby_binary", +) + +package(default_visibility = ["//:__subpackages__"]) + +ruby_binary( + name = "script", + srcs = ["script.rb"], + main = "script.rb", + deps = [ + "@gems//:google-protobuf", + ], +) diff --git a/ruby/tests/testdata/bundle_includes_workspace/Gemfile b/ruby/tests/testdata/bundle_includes_workspace/Gemfile new file mode 100644 index 0000000..5e79722 --- /dev/null +++ b/ruby/tests/testdata/bundle_includes_workspace/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +gem 'google-protobuf' diff --git a/ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock b/ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock new file mode 100644 index 0000000..4499bc1 --- /dev/null +++ b/ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock @@ -0,0 +1,13 @@ +GEM + remote: https://rubygems.org/ + specs: + google-protobuf (3.17.3) + +PLATFORMS + ruby + +DEPENDENCIES + google-protobuf + +BUNDLED WITH + 2.2.21 diff --git a/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE b/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE new file mode 100644 index 0000000..74a33f7 --- /dev/null +++ b/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE @@ -0,0 +1,35 @@ +workspace(name = "bazelruby_rules_ruby_ruby_tests_testdata_bundle_includes_workspace") + +local_repository( + name = "bazelruby_rules_ruby", + path = "../../../..", +) + +load( + "@bazelruby_rules_ruby//ruby:deps.bzl", + "rules_ruby_dependencies", + "rules_ruby_select_sdk", +) + +rules_ruby_dependencies() + +rules_ruby_select_sdk(version = "2.7.1") + +load("@bazelruby_rules_ruby//ruby:defs.bzl", "ruby_bundle") + +ruby_bundle( + name = "gems", + bundler_version = "2.2.21", + gemfile = "//:Gemfile", + gemfile_lock = "//:Gemfile.lock", + includes = { + # The gemspec of google-protobuf rubygem lists ['lib'] as the `require_paths`. Since google-protobuf gem + # dynamically chain-loads different dynamic library files depending upon the major and minor Ruby versions + # at runtime, additional directories and files need to be manually added to `$LOAD_PATH`. + "google-protobuf": [ + "lib/google/*", + "lib/google/2.7/*", + "lib/google/protobuf/**/*", + ], + }, +) diff --git a/ruby/tests/testdata/bundle_includes_workspace/script.rb b/ruby/tests/testdata/bundle_includes_workspace/script.rb new file mode 100644 index 0000000..92d98fe --- /dev/null +++ b/ruby/tests/testdata/bundle_includes_workspace/script.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +expected_gem_require_paths = [ + 'lib', + 'lib/google/*', + 'lib/google/2.7/*', + 'lib/google/protobuf/**/*' +] + +gem_require_paths = $LOAD_PATH.map do |load_path| + %r{.+script.runfiles/gems/lib/ruby/2.7.0/gems/google-protobuf-.+?/(.+)}.match(load_path).to_a[1] +end + +(expected_gem_require_paths - gem_require_paths).each do |missing_require_path| + raise "Expected requir_path '#{missing_require_path}' is missing in $LOAD_PATH." +end + +begin + require 'google/protobuf' +rescue LoadError + $stderr.puts 'Failed to load google-protobuf gem' + raise +end + +puts Google::Protobuf::DescriptorPool.new From 91effc2b8abefe5eedd0fa0c16402503408fa897 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Mon, 5 Jul 2021 06:22:41 +0900 Subject: [PATCH 09/16] tidy up --- ruby/private/bundle/create_bundle_build_file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/private/bundle/create_bundle_build_file.rb b/ruby/private/bundle/create_bundle_build_file.rb index f8f721c..fb90cc1 100755 --- a/ruby/private/bundle/create_bundle_build_file.rb +++ b/ruby/private/bundle/create_bundle_build_file.rb @@ -250,7 +250,7 @@ def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries) warn("registering gem #{spec.name} with binaries: #{gem_binaries}") if bundle_binaries.key?(spec.name) - template_out.puts GEM_TEMPLATE#.gsub('{gem_lib_files}', to_flat_string(gem_lib_dirs.map { |p| "#{p}/**/*" })) + template_out.puts GEM_TEMPLATE .gsub('{gem_lib_dirs}', to_flat_string(gem_lib_dirs)) .gsub('{gem_spec}', spec_path) .gsub('{gem_binaries}', to_flat_string(gem_binaries)) From 07b42905af4e9160fe9df748312ef4d1a3790fdc Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Thu, 8 Jul 2021 22:07:58 +0900 Subject: [PATCH 10/16] Fix the default value of ruby_bundle rule --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9abf7c4..ff83f77 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Note: we have a short guide on [Building your first Ruby Project](https://github.com/bazelruby/rules_ruby/wiki/Build-your-ruby-project) on the Wiki. We encourage you to check it out. -## Table of Contents +## Table of Contents - [Ruby Rules® for Bazel Build System](#ruby-rules-for-bazelhttpsbazelbuild-build-system) - [Build Status & Activity](#build-status-activity) @@ -459,7 +459,7 @@ ruby_test( size, timeout, flaky, - local, + local, shard_count ) ``` @@ -553,7 +553,7 @@ ruby_bundle( gemfile_lock, bundler_version = "2.1.4", includes = {}, - excludes = [], + excludes = {}, vendor_cache = False, ruby_sdk = "@org_ruby_lang_ruby_toolchain", ruby_interpreter = "@org_ruby_lang_ruby_toolchain//:ruby", From 842f8fb50b96cf1ccdd155e405f89f7aa440a395 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Thu, 8 Jul 2021 22:47:04 +0900 Subject: [PATCH 11/16] Bump up Ruby version of ruby_bundle includes sample code --- README.md | 2 +- ruby/tests/testdata/bundle_includes_workspace/WORKSPACE | 4 ++-- ruby/tests/testdata/bundle_includes_workspace/script.rb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ff83f77..90be2f3 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ ruby_bundle( ], "google-protobuf": [ "lib/google/*", - "lib/google/2.7/*", + "lib/google/3.0/*", "lib/google/protobuf/**/*", ] }, diff --git a/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE b/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE index 74a33f7..6b5284d 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE +++ b/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE @@ -13,7 +13,7 @@ load( rules_ruby_dependencies() -rules_ruby_select_sdk(version = "2.7.1") +rules_ruby_select_sdk(version = "3.0.1") load("@bazelruby_rules_ruby//ruby:defs.bzl", "ruby_bundle") @@ -28,7 +28,7 @@ ruby_bundle( # at runtime, additional directories and files need to be manually added to `$LOAD_PATH`. "google-protobuf": [ "lib/google/*", - "lib/google/2.7/*", + "lib/google/3.0/*", "lib/google/protobuf/**/*", ], }, diff --git a/ruby/tests/testdata/bundle_includes_workspace/script.rb b/ruby/tests/testdata/bundle_includes_workspace/script.rb index 92d98fe..0dd370f 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/script.rb +++ b/ruby/tests/testdata/bundle_includes_workspace/script.rb @@ -3,12 +3,12 @@ expected_gem_require_paths = [ 'lib', 'lib/google/*', - 'lib/google/2.7/*', + 'lib/google/3.0/*', 'lib/google/protobuf/**/*' ] gem_require_paths = $LOAD_PATH.map do |load_path| - %r{.+script.runfiles/gems/lib/ruby/2.7.0/gems/google-protobuf-.+?/(.+)}.match(load_path).to_a[1] + %r{.+script.runfiles/gems/lib/ruby/3.0.1/gems/google-protobuf-.+?/(.+)}.match(load_path).to_a[1] end (expected_gem_require_paths - gem_require_paths).each do |missing_require_path| From 3f1952eae9f206412adc1eb437193a8767fe94b5 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Fri, 9 Jul 2021 05:43:05 +0900 Subject: [PATCH 12/16] Fix new test workspace --- WORKSPACE | 5 +++++ ruby/tests/testdata/bundle_includes_workspace/script.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 20e7927..0ebc4dd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -19,6 +19,11 @@ local_repository( path = "ruby/tests/testdata/another_workspace", ) +local_repository( + name = "bazelruby_rules_ruby_ruby_tests_testdata_bundle_includes_workspace", + path = "ruby/tests/testdata/bundle_includes_workspace", +) + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") diff --git a/ruby/tests/testdata/bundle_includes_workspace/script.rb b/ruby/tests/testdata/bundle_includes_workspace/script.rb index 0dd370f..249659c 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/script.rb +++ b/ruby/tests/testdata/bundle_includes_workspace/script.rb @@ -8,7 +8,7 @@ ] gem_require_paths = $LOAD_PATH.map do |load_path| - %r{.+script.runfiles/gems/lib/ruby/3.0.1/gems/google-protobuf-.+?/(.+)}.match(load_path).to_a[1] + %r{.+script.runfiles/(?:gems|bundle)/lib/ruby/3.0.0/gems/google-protobuf-.+?/(.+)}.match(load_path).to_a[1] end (expected_gem_require_paths - gem_require_paths).each do |missing_require_path| From 6b907c01852e2552d8445bd213877434def6be1d Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Fri, 9 Jul 2021 06:56:32 +0900 Subject: [PATCH 13/16] Fix bundle build rule path injection --- ruby/private/bundle/create_bundle_build_file.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ruby/private/bundle/create_bundle_build_file.rb b/ruby/private/bundle/create_bundle_build_file.rb index 2a72b44..db40418 100755 --- a/ruby/private/bundle/create_bundle_build_file.rb +++ b/ruby/private/bundle/create_bundle_build_file.rb @@ -33,14 +33,14 @@ srcs = glob( include = [ ".bundle/config", - {gem_lib_dirs}, + {gem_lib_files}, "{gem_spec}", {gem_binaries} ], exclude = {exclude}, ), deps = {deps}, - includes = [{gem_lib_dirs}], + includes = [{gem_lib_paths}], ) GEM_TEMPLATE @@ -238,12 +238,12 @@ def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries) base_dir = "lib/ruby/#{ruby_version}" # paths to register to $LOAD_PATH - # Usually, registering the directory paths listed in the `require_paths` of gemspecs is sufficient, but - # some rubygems also require additional paths to be included in the load paths. require_paths = Gem::StubSpecification.gemspec_stub(spec_path, base_dir, "#{base_dir}/gems").require_paths + # Usually, registering the directory paths listed in the `require_paths` of gemspecs is sufficient, but + # some gems also require additional paths to be included in the load paths. require_paths += include_array(spec.name) - gem_lib_dirs = require_paths.map { |require_path| File.join(gem_path, require_path) } - bundle_lib_paths += gem_lib_dirs + gem_lib_paths = require_paths.map { |require_path| File.join(gem_path, require_path) } + bundle_lib_paths.push(*gem_lib_paths) # paths to search for executables gem_binaries = find_bundle_binaries(gem_path) @@ -254,7 +254,8 @@ def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries) warn("registering gem #{spec.name} with binaries: #{gem_binaries}") if bundle_binaries.key?(spec.name) template_out.puts GEM_TEMPLATE - .gsub('{gem_lib_dirs}', to_flat_string(gem_lib_dirs)) + .gsub('{gem_lib_paths}', to_flat_string(gem_lib_paths)) + .gsub('{gem_lib_files}', to_flat_string(gem_lib_paths.map { |p| "#{p}/**/*" })) .gsub('{gem_spec}', spec_path) .gsub('{gem_binaries}', to_flat_string(gem_binaries)) .gsub('{exclude}', exclude_array(spec.name).to_s) From 13ab55e0cd83de0e7fe095adaaa69214f127e3ac Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Fri, 9 Jul 2021 07:45:44 +0900 Subject: [PATCH 14/16] Fix usage of ruby_bundle rule's includes option --- README.md | 10 +--------- .../bundle_includes_workspace/BUILD.bazel | 2 +- .../testdata/bundle_includes_workspace/Gemfile | 2 +- .../bundle_includes_workspace/Gemfile.lock | 9 +++++++-- .../bundle_includes_workspace/WORKSPACE | 13 +++++-------- .../bundle_includes_workspace/script.rb | 18 +++++++++--------- 6 files changed, 24 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 90be2f3..62ed41c 100644 --- a/README.md +++ b/README.md @@ -101,15 +101,7 @@ ruby_bundle( name = "bundle", # Specify additional paths to be loaded from the gems at runtime, if any. includes = { - "grpc": [ - "etc/*", - "src/ruby/lib/*", - ], - "google-protobuf": [ - "lib/google/*", - "lib/google/3.0/*", - "lib/google/protobuf/**/*", - ] + "grpc": ["etc"], }, excludes = { "mini_portile": ["test/**/*"], diff --git a/ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel b/ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel index 4605a35..5df776a 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel +++ b/ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel @@ -10,6 +10,6 @@ ruby_binary( srcs = ["script.rb"], main = "script.rb", deps = [ - "@gems//:google-protobuf", + "@gems//:grpc", ], ) diff --git a/ruby/tests/testdata/bundle_includes_workspace/Gemfile b/ruby/tests/testdata/bundle_includes_workspace/Gemfile index 5e79722..9595a03 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/Gemfile +++ b/ruby/tests/testdata/bundle_includes_workspace/Gemfile @@ -2,4 +2,4 @@ source 'https://rubygems.org' -gem 'google-protobuf' +gem 'grpc' diff --git a/ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock b/ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock index 4499bc1..0417ff0 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock +++ b/ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock @@ -2,12 +2,17 @@ GEM remote: https://rubygems.org/ specs: google-protobuf (3.17.3) + googleapis-common-protos-types (1.1.0) + google-protobuf (~> 3.14) + grpc (1.38.0) + google-protobuf (~> 3.15) + googleapis-common-protos-types (~> 1.0) PLATFORMS ruby DEPENDENCIES - google-protobuf + grpc BUNDLED WITH - 2.2.21 + 2.2.22 diff --git a/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE b/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE index 6b5284d..4beeae4 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE +++ b/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE @@ -23,13 +23,10 @@ ruby_bundle( gemfile = "//:Gemfile", gemfile_lock = "//:Gemfile.lock", includes = { - # The gemspec of google-protobuf rubygem lists ['lib'] as the `require_paths`. Since google-protobuf gem - # dynamically chain-loads different dynamic library files depending upon the major and minor Ruby versions - # at runtime, additional directories and files need to be manually added to `$LOAD_PATH`. - "google-protobuf": [ - "lib/google/*", - "lib/google/3.0/*", - "lib/google/protobuf/**/*", - ], + # The gemspec of grpc gem lists ['src/ruby/bin', 'src/ruby/lib', 'src/ruby/pb'] as the `require_paths`. When installing + # pre-built versions of the gem, these are sufficient. However, when installing it from source, 'etc/roots.pem', a file + # required by 'src/ruby/lib/grpc.rb', also needs to be present in the `$LOAD_PATH`. Thus users have to manually add the + # 'etc' directory to the `$LOAD_PATH` using the `includes` option of `ruby_bundle` rule. + "grpc": ["etc"], }, ) diff --git a/ruby/tests/testdata/bundle_includes_workspace/script.rb b/ruby/tests/testdata/bundle_includes_workspace/script.rb index 249659c..c29eab5 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/script.rb +++ b/ruby/tests/testdata/bundle_includes_workspace/script.rb @@ -1,25 +1,25 @@ # frozen_string_literal: true expected_gem_require_paths = [ - 'lib', - 'lib/google/*', - 'lib/google/3.0/*', - 'lib/google/protobuf/**/*' + 'etc', + 'src/ruby/bin', + 'src/ruby/lib', + 'src/ruby/pb' ] gem_require_paths = $LOAD_PATH.map do |load_path| - %r{.+script.runfiles/(?:gems|bundle)/lib/ruby/3.0.0/gems/google-protobuf-.+?/(.+)}.match(load_path).to_a[1] -end + %r{.+script.runfiles/(?:gems|bundle)/lib/ruby/3.0.0/gems/grpc-.+?/(.+)}.match(load_path).to_a[1] +end.compact (expected_gem_require_paths - gem_require_paths).each do |missing_require_path| raise "Expected requir_path '#{missing_require_path}' is missing in $LOAD_PATH." end begin - require 'google/protobuf' + require 'grpc' rescue LoadError - $stderr.puts 'Failed to load google-protobuf gem' + $stderr.puts 'Failed to load grpc gem' raise end -puts Google::Protobuf::DescriptorPool.new +puts GRPC::RpcServer.new From 240e161cad257663c49d300ff0fc29eaa803ef64 Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Fri, 9 Jul 2021 08:00:46 +0900 Subject: [PATCH 15/16] More clarification in comments --- ruby/tests/testdata/bundle_includes_workspace/WORKSPACE | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE b/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE index 4beeae4..c97411c 100644 --- a/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE +++ b/ruby/tests/testdata/bundle_includes_workspace/WORKSPACE @@ -24,9 +24,12 @@ ruby_bundle( gemfile_lock = "//:Gemfile.lock", includes = { # The gemspec of grpc gem lists ['src/ruby/bin', 'src/ruby/lib', 'src/ruby/pb'] as the `require_paths`. When installing - # pre-built versions of the gem, these are sufficient. However, when installing it from source, 'etc/roots.pem', a file - # required by 'src/ruby/lib/grpc.rb', also needs to be present in the `$LOAD_PATH`. Thus users have to manually add the - # 'etc' directory to the `$LOAD_PATH` using the `includes` option of `ruby_bundle` rule. + # pre-built versions of the gem using a package downloaded from rubygems.org, these paths are sufficient since the file + # `src/ruby/lib/grpc.rb` in the downloaded gem package does not `require` any file outside these directories. + # However, when installing grpc gem from source using Bundler, `src/ruby/lib/grpc.rb` in the source package does + # `require` 'etc/roots.pem', so the directory containing this `require`-d file also needs to be present in the `$LOAD_PATH`. + # Thus users have to manually add the 'etc' directory to the `$LOAD_PATH` using the `includes` option of `ruby_bundle` rule. + # The `includes` option of `ruby_bundle` rule is a means of workaround for such a peculiar situation. "grpc": ["etc"], }, ) From 11cbada9cd83d7245009bb553601316118dec2ac Mon Sep 17 00:00:00 2001 From: Minoru Mizutani Date: Sat, 10 Jul 2021 06:12:51 +0900 Subject: [PATCH 16/16] Clarify that folders in spec.require_paths do not need to be listed in includes hash --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 62ed41c..fb831c3 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ load( ruby_bundle( name = "bundle", # Specify additional paths to be loaded from the gems at runtime, if any. + # Since spec.require_paths in Gem specifications are auto-included, directory paths + # in spec.require_paths do not need to be listed in includes hash. includes = { "grpc": ["etc"], },