Merge branch 'protocolbuffers:main' into improve-performance-repeated-packed-fixed-field

Author: YarinOmesi

.github/workflows/test_release_branches.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: [25.x, 28.x, 29.x]
+        branch: [25.x, 29.x]
     runs-on: ubuntu-latest
     permissions:
       actions: write

.github/workflows/test_upb.yml

@@ -264,13 +264,24 @@ jobs:
       - name: Install Protobuf Test Wheel
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}
         run: pip install -vvv --no-index --find-links wheels protobuftests
-      - name: Run the unit tests
+      - name: Run legacy unit tests
         if: ${{ !matrix.continuous-only || inputs.continuous-run }}
+        # Legacy upb tests use a "test wrapper" and get installed under pb_unit_tests.
         run: |
-          TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g')
-          for test in $TESTS; do
+          PB_UNIT_TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g')
+          for test in $PB_UNIT_TESTS; do
             python -m unittest -v $test
           done
+      - name: Run unit tests
+        if: ${{ !matrix.continuous-only || inputs.continuous-run }}
+        # Newer upb tests are in the standard google.protobuf.internal path.
+        # We will eventually make this into a wildcard rule once all tests
+        # have been migrated to be compatible with upb.
+        run: |
+          TESTS=(message_test message_factory_test descriptor_test proto_builder_test)
+          for test in ${TESTS[@]}; do
+            python -m unittest -v google.protobuf.internal.${test}
+          done
 
   test_pure_python_wheels:
     name: Test Pure Python Wheels
@@ -299,8 +310,13 @@ jobs:
           python -m venv env
           source env/bin/activate
           echo "VIRTUAL ENV:" $VIRTUAL_ENV
-      - name: Install numpy
-        run: pip install numpy
+      - name: Download Requirements
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #4.1.8
+        with:
+          name: requirements
+          path: requirements
+      - name: Install requirements
+        run: pip install -r requirements/requirements.txt
       - name: Install Protobuf Wheels
         run: pip install -vvv --no-index --find-links wheels protobuf protobuftests
       - name: Run the unit tests

.gitignore

@@ -176,6 +176,11 @@ ruby/tests/multi_level_nesting_test_pb.rb
 ruby/tests/service_test_pb.rb
 ruby/tests/test_import_proto2_pb.rb
 ruby/tests/test_ruby_package_proto2_pb.rb
+ruby/tests/basic_test_features_pb.rb
+ruby/tests/generated_code_editions_pb.rb
+ruby/tests/repeated_field_test_pb.rb
+ruby/tests/stress_pb.rb
+ruby/tests/utf8_pb.rb
 ruby/compatibility_tests/v3.0.0/protoc
 ruby/compatibility_tests/v3.0.0/tests/generated_code_pb.rb
 ruby/compatibility_tests/v3.0.0/tests/test_import_pb.rb

MODULE.bazel

@@ -58,7 +58,7 @@ use_repo(
     system_python = "python_{}".format(SUPPORTED_PYTHON_VERSIONS[-1].replace(".", "_")),
 )
 
-pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True)
 
 [
     pip.parse(
@@ -71,7 +71,7 @@ pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 
 use_repo(pip, "pip_deps")
 
-crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
+crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate", dev_dependency = True)
 crate.spec(
     package = "googletest",
     version = ">0.0.0",
@@ -83,7 +83,7 @@ crate.spec(
 crate.from_specs()
 use_repo(crate, crate_index = "crates")
 
-maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
+maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven", dev_dependency = True)
 maven.install(
     name = "protobuf_maven",
     artifacts = [
@@ -112,6 +112,12 @@ use_repo(maven, "protobuf_maven")
 bazel_dep(name = "googletest", version = "1.14.0", dev_dependency = True, repo_name = "com_google_googletest")
 bazel_dep(name = "rules_buf", version = "0.3.0", dev_dependency = True)
 bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True)
+bazel_dep(
+    name = "abseil-py",
+    version = "2.1.0",
+    dev_dependency = True,
+    repo_name = "com_google_absl_py",
+)
 
 # rules_proto are needed for @com_google_protobuf_v25.0 used in //compatibility/... tests
 bazel_dep(name = "rules_proto", version = "4.0.0", dev_dependency = True)

WORKSPACE

@@ -193,6 +193,15 @@ load("@pip_deps//:requirements.bzl", "install_deps")
 
 install_deps()
 
+http_archive(
+    name = "com_google_absl_py",
+    sha256 = "8a3d0830e4eb4f66c4fa907c06edf6ce1c719ced811a12e26d9d3162f8471758",
+    strip_prefix = "abseil-py-2.1.0",
+    urls = [
+        "https://github.com/abseil/abseil-py/archive/refs/tags/v2.1.0.tar.gz",
+    ],
+)
+
 http_archive(
     name = "rules_fuzzing",
     patch_args = ["-p1"],

cmake/dependencies.cmake

@@ -30,6 +30,7 @@ set(rules_shell-version "0.2.0")
 set(googletest-version "1.14.0")
 set(rules_buf-version "0.3.0")
 set(rules_testing-version "0.6.0")
+set(abseil-py-version "2.1.0")
 set(rules_proto-version "4.0.0")
 
 

cmake/dependencies_generator.py

@@ -90,7 +90,7 @@ def register_toolchains(self, *args):
   def use_repo(self, *args, **kwargs):
     pass
 
-  def use_extension(self, *args):
+  def use_extension(self, *args, **kwargs):
     return ExtensionFunctions()
 
 

cmake/install.cmake

@@ -93,6 +93,7 @@ set(protobuf_HEADERS
   ${descriptor_proto_proto_srcs}
   ${plugin_proto_proto_srcs}
   ${java_features_proto_proto_srcs}
+  ${go_features_proto_proto_srcs}
 )
 if (protobuf_BUILD_LIBUPB)
   list(APPEND protobuf_HEADERS ${libupb_hdrs})
@@ -105,14 +106,22 @@ if (protobuf_BUILD_LIBUPB)
     COMPONENT protobuf-headers
   )
 endif ()
+set(protobuf_STRIP_PREFIXES
+  "/src"
+  "/java/core/src/main/resources"
+  "/go"
+  "/"
+)
 foreach(_header ${protobuf_HEADERS})
-  string(FIND ${_header} "${protobuf_SOURCE_DIR}/src" _find_src)
-  string(FIND ${_header} "${protobuf_SOURCE_DIR}" _find_nosrc)
-  if (_find_src GREATER -1)
-    set(_from_dir "${protobuf_SOURCE_DIR}/src")
-  elseif (_find_nosrc GREATER -1)
-    set(_from_dir "${protobuf_SOURCE_DIR}")
-  endif()
+  foreach(_strip_prefix ${protobuf_STRIP_PREFIXES})
+    string(FIND ${_header} "${protobuf_SOURCE_DIR}${_strip_prefix}" _find_src)
+    if(_find_src GREATER -1)
+      set(_from_dir "${protobuf_SOURCE_DIR}${_strip_prefix}")
+      break()
+    endif()
+  endforeach()
+  message(${_from_dir} "-" ${_header})
+
   # Escape _from_dir for regex special characters in the directory name.
   string(REGEX REPLACE "([$^.[|*+?()]|])" "\\\\\\1" _from_dir_regexp "${_from_dir}")
   # On some platforms `_form_dir` ends up being just "protobuf", which can

conformance/BUILD.bazel

@@ -1,3 +1,10 @@
+# Copyright (c) 2024, Google LLC
+# All rights reserved.
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
 # Conformance testing for Protobuf.
 
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "objc_library")

conformance/text_format_failure_list_python.txt

@@ -1,8 +1,6 @@
 # This is the list of text format conformance tests that are known to fail right
 # now.
 # TODO: These should be fixed.
-Required.*.TextFormatInput.FloatFieldNoNegativeOctal                                                               # Should have failed to parse, but didn't.
-Required.*.TextFormatInput.FloatFieldNoOctal                                                                       # Should have failed to parse, but didn't.
 Required.*.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput                                           # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t
 Required.*.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput                                         # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t
 Required.*.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput                                          # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\